2024-12-03 17:15:44,226 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-03 17:15:44,252 main DEBUG Took 0.023581 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-03 17:15:44,253 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-03 17:15:44,255 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-03 17:15:44,256 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-03 17:15:44,257 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 17:15:44,274 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-03 17:15:44,304 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 17:15:44,306 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 17:15:44,307 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 17:15:44,308 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 17:15:44,308 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 17:15:44,309 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 17:15:44,310 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 17:15:44,310 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 17:15:44,312 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 17:15:44,312 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 17:15:44,313 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 17:15:44,313 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 17:15:44,314 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 17:15:44,315 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 17:15:44,316 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 17:15:44,316 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 17:15:44,317 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 17:15:44,317 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 17:15:44,318 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 17:15:44,318 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 17:15:44,318 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 17:15:44,319 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 17:15:44,319 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 17:15:44,320 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 17:15:44,321 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 17:15:44,321 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-03 17:15:44,323 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 17:15:44,325 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-03 17:15:44,328 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-03 17:15:44,328 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-03 17:15:44,330 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-03 17:15:44,330 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-03 17:15:44,343 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-03 17:15:44,347 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-03 17:15:44,349 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-03 17:15:44,350 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-03 17:15:44,351 main DEBUG createAppenders(={Console}) 2024-12-03 17:15:44,352 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-12-03 17:15:44,352 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-03 17:15:44,353 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-12-03 17:15:44,353 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-03 17:15:44,354 main DEBUG OutputStream closed 2024-12-03 17:15:44,354 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-03 17:15:44,354 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-03 17:15:44,355 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-12-03 17:15:44,505 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-03 17:15:44,513 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-03 17:15:44,515 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-03 17:15:44,516 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-03 17:15:44,517 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-03 17:15:44,520 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-03 17:15:44,521 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-03 17:15:44,522 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-03 17:15:44,522 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-03 17:15:44,523 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-03 17:15:44,524 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-03 17:15:44,524 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-03 17:15:44,525 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-03 17:15:44,525 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-03 17:15:44,526 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-03 17:15:44,526 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-03 17:15:44,526 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-03 17:15:44,528 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-03 17:15:44,532 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-03 17:15:44,533 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-12-03 17:15:44,533 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-03 17:15:44,534 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-12-03T17:15:44,961 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f784850d-ec8c-cbd4-e545-debb5ca850cc 2024-12-03 17:15:44,966 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-03 17:15:44,966 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-03T17:15:44,979 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-12-03T17:15:45,011 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-03T17:15:45,043 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f784850d-ec8c-cbd4-e545-debb5ca850cc/cluster_481ba349-0fa7-2aa3-a4cc-d5a666163a05, deleteOnExit=true 2024-12-03T17:15:45,044 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-03T17:15:45,045 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f784850d-ec8c-cbd4-e545-debb5ca850cc/test.cache.data in system properties and HBase conf 2024-12-03T17:15:45,046 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f784850d-ec8c-cbd4-e545-debb5ca850cc/hadoop.tmp.dir in system properties and HBase conf 2024-12-03T17:15:45,046 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f784850d-ec8c-cbd4-e545-debb5ca850cc/hadoop.log.dir in system properties and HBase conf 2024-12-03T17:15:45,047 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f784850d-ec8c-cbd4-e545-debb5ca850cc/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-03T17:15:45,048 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f784850d-ec8c-cbd4-e545-debb5ca850cc/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-03T17:15:45,048 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-03T17:15:45,327 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-03T17:15:45,504 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-03T17:15:45,521 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f784850d-ec8c-cbd4-e545-debb5ca850cc/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-03T17:15:45,534 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f784850d-ec8c-cbd4-e545-debb5ca850cc/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-03T17:15:45,538 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f784850d-ec8c-cbd4-e545-debb5ca850cc/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-03T17:15:45,539 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f784850d-ec8c-cbd4-e545-debb5ca850cc/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-03T17:15:45,540 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f784850d-ec8c-cbd4-e545-debb5ca850cc/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-03T17:15:45,540 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f784850d-ec8c-cbd4-e545-debb5ca850cc/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-03T17:15:45,541 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f784850d-ec8c-cbd4-e545-debb5ca850cc/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-03T17:15:45,543 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f784850d-ec8c-cbd4-e545-debb5ca850cc/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-03T17:15:45,544 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f784850d-ec8c-cbd4-e545-debb5ca850cc/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-03T17:15:45,545 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f784850d-ec8c-cbd4-e545-debb5ca850cc/nfs.dump.dir in system properties and HBase conf 2024-12-03T17:15:45,545 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f784850d-ec8c-cbd4-e545-debb5ca850cc/java.io.tmpdir in system properties and HBase conf 2024-12-03T17:15:45,546 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f784850d-ec8c-cbd4-e545-debb5ca850cc/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-03T17:15:45,547 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f784850d-ec8c-cbd4-e545-debb5ca850cc/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-03T17:15:45,549 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f784850d-ec8c-cbd4-e545-debb5ca850cc/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-03T17:15:46,849 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-03T17:15:46,960 INFO [Time-limited test {}] log.Log(170): Logging initialized @3763ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-03T17:15:47,078 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T17:15:47,168 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T17:15:47,217 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T17:15:47,217 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T17:15:47,219 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-03T17:15:47,236 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T17:15:47,240 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f784850d-ec8c-cbd4-e545-debb5ca850cc/hadoop.log.dir/,AVAILABLE} 2024-12-03T17:15:47,242 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T17:15:47,522 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f784850d-ec8c-cbd4-e545-debb5ca850cc/java.io.tmpdir/jetty-localhost-34711-hadoop-hdfs-3_4_1-tests_jar-_-any-8398760333592730782/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-03T17:15:47,536 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:34711} 2024-12-03T17:15:47,536 INFO [Time-limited test {}] server.Server(415): Started @4341ms 2024-12-03T17:15:48,227 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T17:15:48,236 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T17:15:48,239 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T17:15:48,239 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T17:15:48,240 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-03T17:15:48,241 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f784850d-ec8c-cbd4-e545-debb5ca850cc/hadoop.log.dir/,AVAILABLE} 2024-12-03T17:15:48,242 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T17:15:48,393 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f784850d-ec8c-cbd4-e545-debb5ca850cc/java.io.tmpdir/jetty-localhost-42361-hadoop-hdfs-3_4_1-tests_jar-_-any-11981180918606607902/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T17:15:48,394 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:42361} 2024-12-03T17:15:48,394 INFO [Time-limited test {}] server.Server(415): Started @5199ms 2024-12-03T17:15:48,460 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-03T17:15:49,158 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f784850d-ec8c-cbd4-e545-debb5ca850cc/cluster_481ba349-0fa7-2aa3-a4cc-d5a666163a05/data/data1/current/BP-2063810512-172.17.0.3-1733246146518/current, will proceed with Du for space computation calculation, 2024-12-03T17:15:49,158 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f784850d-ec8c-cbd4-e545-debb5ca850cc/cluster_481ba349-0fa7-2aa3-a4cc-d5a666163a05/data/data2/current/BP-2063810512-172.17.0.3-1733246146518/current, will proceed with Du for space computation calculation, 2024-12-03T17:15:49,274 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-03T17:15:49,359 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x13042d2948cfd7bf with lease ID 0x88ae576570c85274: Processing first storage report for DS-1f8e4f8f-fe3f-41b1-8213-7117a2e64c04 from datanode DatanodeRegistration(127.0.0.1:38185, datanodeUuid=e44d842c-8e46-41fc-93b6-c38882841a8c, infoPort=39969, infoSecurePort=0, ipcPort=46073, storageInfo=lv=-57;cid=testClusterID;nsid=816095180;c=1733246146518) 2024-12-03T17:15:49,362 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x13042d2948cfd7bf with lease ID 0x88ae576570c85274: from storage DS-1f8e4f8f-fe3f-41b1-8213-7117a2e64c04 node DatanodeRegistration(127.0.0.1:38185, datanodeUuid=e44d842c-8e46-41fc-93b6-c38882841a8c, infoPort=39969, infoSecurePort=0, ipcPort=46073, storageInfo=lv=-57;cid=testClusterID;nsid=816095180;c=1733246146518), blocks: 0, hasStaleStorage: true, processing time: 4 msecs, invalidatedBlocks: 0 2024-12-03T17:15:49,363 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x13042d2948cfd7bf with lease ID 0x88ae576570c85274: Processing first storage report for DS-fa03943d-288f-43df-9caa-41014ea32e8a from datanode DatanodeRegistration(127.0.0.1:38185, datanodeUuid=e44d842c-8e46-41fc-93b6-c38882841a8c, infoPort=39969, infoSecurePort=0, ipcPort=46073, storageInfo=lv=-57;cid=testClusterID;nsid=816095180;c=1733246146518) 2024-12-03T17:15:49,363 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x13042d2948cfd7bf with lease ID 0x88ae576570c85274: from storage DS-fa03943d-288f-43df-9caa-41014ea32e8a node DatanodeRegistration(127.0.0.1:38185, datanodeUuid=e44d842c-8e46-41fc-93b6-c38882841a8c, infoPort=39969, infoSecurePort=0, ipcPort=46073, storageInfo=lv=-57;cid=testClusterID;nsid=816095180;c=1733246146518), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-03T17:15:49,405 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f784850d-ec8c-cbd4-e545-debb5ca850cc 2024-12-03T17:15:49,535 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f784850d-ec8c-cbd4-e545-debb5ca850cc/cluster_481ba349-0fa7-2aa3-a4cc-d5a666163a05/zookeeper_0, clientPort=56976, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f784850d-ec8c-cbd4-e545-debb5ca850cc/cluster_481ba349-0fa7-2aa3-a4cc-d5a666163a05/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f784850d-ec8c-cbd4-e545-debb5ca850cc/cluster_481ba349-0fa7-2aa3-a4cc-d5a666163a05/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-03T17:15:49,554 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=56976 2024-12-03T17:15:49,568 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T17:15:49,572 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T17:15:49,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741825_1001 (size=7) 2024-12-03T17:15:50,355 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4 with version=8 2024-12-03T17:15:50,356 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/hbase-staging 2024-12-03T17:15:50,510 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-03T17:15:50,837 INFO [Time-limited test {}] client.ConnectionUtils(128): master/27e0f852a164:0 server-side Connection retries=45 2024-12-03T17:15:50,851 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T17:15:50,852 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T17:15:50,859 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T17:15:50,859 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T17:15:50,860 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T17:15:51,050 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-03T17:15:51,127 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-03T17:15:51,140 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-03T17:15:51,145 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T17:15:51,183 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 2779 (auto-detected) 2024-12-03T17:15:51,184 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-12-03T17:15:51,209 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:38523 2024-12-03T17:15:51,239 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38523 connecting to ZooKeeper ensemble=127.0.0.1:56976 2024-12-03T17:15:51,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:385230x0, quorum=127.0.0.1:56976, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T17:15:51,292 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38523-0x1010d48f3a20000 connected 2024-12-03T17:15:51,396 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T17:15:51,400 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T17:15:51,423 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38523-0x1010d48f3a20000, quorum=127.0.0.1:56976, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T17:15:51,428 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4, hbase.cluster.distributed=false 2024-12-03T17:15:51,471 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38523-0x1010d48f3a20000, quorum=127.0.0.1:56976, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T17:15:51,480 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38523 2024-12-03T17:15:51,488 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38523 2024-12-03T17:15:51,490 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38523 2024-12-03T17:15:51,491 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38523 2024-12-03T17:15:51,492 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38523 2024-12-03T17:15:51,652 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/27e0f852a164:0 server-side Connection retries=45 2024-12-03T17:15:51,655 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T17:15:51,656 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T17:15:51,656 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T17:15:51,656 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T17:15:51,657 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T17:15:51,661 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-03T17:15:51,665 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T17:15:51,668 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:41167 2024-12-03T17:15:51,671 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41167 connecting to ZooKeeper ensemble=127.0.0.1:56976 2024-12-03T17:15:51,674 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T17:15:51,678 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T17:15:51,691 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:411670x0, quorum=127.0.0.1:56976, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T17:15:51,692 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41167-0x1010d48f3a20001 connected 2024-12-03T17:15:51,692 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41167-0x1010d48f3a20001, quorum=127.0.0.1:56976, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T17:15:51,699 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-03T17:15:51,711 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-03T17:15:51,716 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41167-0x1010d48f3a20001, quorum=127.0.0.1:56976, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-03T17:15:51,722 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41167-0x1010d48f3a20001, quorum=127.0.0.1:56976, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T17:15:51,727 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41167 2024-12-03T17:15:51,729 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41167 2024-12-03T17:15:51,730 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41167 2024-12-03T17:15:51,734 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41167 2024-12-03T17:15:51,737 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41167 2024-12-03T17:15:51,759 DEBUG [M:0;27e0f852a164:38523 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;27e0f852a164:38523 2024-12-03T17:15:51,760 INFO [master/27e0f852a164:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/27e0f852a164,38523,1733246150595 2024-12-03T17:15:51,769 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38523-0x1010d48f3a20000, quorum=127.0.0.1:56976, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T17:15:51,769 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41167-0x1010d48f3a20001, quorum=127.0.0.1:56976, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T17:15:51,771 DEBUG [master/27e0f852a164:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38523-0x1010d48f3a20000, quorum=127.0.0.1:56976, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/27e0f852a164,38523,1733246150595 2024-12-03T17:15:51,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41167-0x1010d48f3a20001, quorum=127.0.0.1:56976, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-03T17:15:51,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38523-0x1010d48f3a20000, quorum=127.0.0.1:56976, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T17:15:51,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41167-0x1010d48f3a20001, quorum=127.0.0.1:56976, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T17:15:51,799 DEBUG [master/27e0f852a164:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38523-0x1010d48f3a20000, quorum=127.0.0.1:56976, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-03T17:15:51,806 INFO [master/27e0f852a164:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/27e0f852a164,38523,1733246150595 from backup master directory 2024-12-03T17:15:51,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38523-0x1010d48f3a20000, quorum=127.0.0.1:56976, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/27e0f852a164,38523,1733246150595 2024-12-03T17:15:51,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41167-0x1010d48f3a20001, quorum=127.0.0.1:56976, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T17:15:51,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38523-0x1010d48f3a20000, quorum=127.0.0.1:56976, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T17:15:51,813 WARN [master/27e0f852a164:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T17:15:51,813 INFO [master/27e0f852a164:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=27e0f852a164,38523,1733246150595 2024-12-03T17:15:51,817 INFO [master/27e0f852a164:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-03T17:15:51,819 INFO [master/27e0f852a164:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-03T17:15:51,921 DEBUG [master/27e0f852a164:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/hbase.id] with ID: ab945027-d9b6-4a24-a77e-0e4db28ff7fe 2024-12-03T17:15:51,921 DEBUG [master/27e0f852a164:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/.tmp/hbase.id 2024-12-03T17:15:51,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741826_1002 (size=42) 2024-12-03T17:15:52,367 DEBUG [master/27e0f852a164:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/.tmp/hbase.id]:[hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/hbase.id] 2024-12-03T17:15:52,430 INFO [master/27e0f852a164:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T17:15:52,436 INFO [master/27e0f852a164:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-03T17:15:52,464 INFO [master/27e0f852a164:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 25ms. 2024-12-03T17:15:52,469 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41167-0x1010d48f3a20001, quorum=127.0.0.1:56976, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T17:15:52,469 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38523-0x1010d48f3a20000, quorum=127.0.0.1:56976, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T17:15:52,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741827_1003 (size=196) 2024-12-03T17:15:52,918 INFO [master/27e0f852a164:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T17:15:52,921 INFO [master/27e0f852a164:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-03T17:15:52,944 DEBUG [master/27e0f852a164:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:15:52,951 INFO [master/27e0f852a164:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T17:15:52,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741828_1004 (size=1189) 2024-12-03T17:15:53,425 INFO [master/27e0f852a164:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/MasterData/data/master/store 2024-12-03T17:15:53,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741829_1005 (size=34) 2024-12-03T17:15:53,463 INFO [master/27e0f852a164:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-03T17:15:53,467 DEBUG [master/27e0f852a164:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T17:15:53,469 DEBUG [master/27e0f852a164:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-03T17:15:53,470 INFO [master/27e0f852a164:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T17:15:53,470 DEBUG [master/27e0f852a164:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T17:15:53,473 DEBUG [master/27e0f852a164:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-03T17:15:53,473 DEBUG [master/27e0f852a164:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T17:15:53,474 INFO [master/27e0f852a164:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T17:15:53,476 DEBUG [master/27e0f852a164:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733246153469Disabling compacts and flushes for region at 1733246153469Disabling writes for close at 1733246153473 (+4 ms)Writing region close event to WAL at 1733246153473Closed at 1733246153474 (+1 ms) 2024-12-03T17:15:53,479 WARN [master/27e0f852a164:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/MasterData/data/master/store/.initializing 2024-12-03T17:15:53,480 DEBUG [master/27e0f852a164:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/MasterData/WALs/27e0f852a164,38523,1733246150595 2024-12-03T17:15:53,493 INFO [master/27e0f852a164:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T17:15:53,516 INFO [master/27e0f852a164:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=27e0f852a164%2C38523%2C1733246150595, suffix=, logDir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/MasterData/WALs/27e0f852a164,38523,1733246150595, archiveDir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/MasterData/oldWALs, maxLogs=10 2024-12-03T17:15:53,546 DEBUG [master/27e0f852a164:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/MasterData/WALs/27e0f852a164,38523,1733246150595/27e0f852a164%2C38523%2C1733246150595.1733246153522, exclude list is [], retry=0 2024-12-03T17:15:53,572 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38185,DS-1f8e4f8f-fe3f-41b1-8213-7117a2e64c04,DISK] 2024-12-03T17:15:53,576 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-03T17:15:53,632 INFO [master/27e0f852a164:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/MasterData/WALs/27e0f852a164,38523,1733246150595/27e0f852a164%2C38523%2C1733246150595.1733246153522 2024-12-03T17:15:53,636 DEBUG [master/27e0f852a164:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39969:39969)] 2024-12-03T17:15:53,637 DEBUG [master/27e0f852a164:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-03T17:15:53,638 DEBUG [master/27e0f852a164:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T17:15:53,643 DEBUG [master/27e0f852a164:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T17:15:53,646 DEBUG [master/27e0f852a164:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T17:15:53,736 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T17:15:53,790 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-03T17:15:53,796 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:15:53,805 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T17:15:53,806 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T17:15:53,819 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-03T17:15:53,819 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:15:53,823 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T17:15:53,824 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T17:15:53,828 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-03T17:15:53,829 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:15:53,831 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T17:15:53,831 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T17:15:53,836 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-03T17:15:53,837 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:15:53,838 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T17:15:53,839 DEBUG [master/27e0f852a164:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T17:15:53,845 DEBUG [master/27e0f852a164:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-03T17:15:53,847 DEBUG [master/27e0f852a164:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-03T17:15:53,857 DEBUG [master/27e0f852a164:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T17:15:53,858 DEBUG [master/27e0f852a164:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T17:15:53,864 DEBUG [master/27e0f852a164:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T17:15:53,870 DEBUG [master/27e0f852a164:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T17:15:53,876 DEBUG [master/27e0f852a164:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T17:15:53,878 INFO [master/27e0f852a164:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67747310, jitterRate=0.00951358675956726}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T17:15:53,884 DEBUG [master/27e0f852a164:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733246153666Initializing all the Stores at 1733246153686 (+20 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733246153687 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733246153690 (+3 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733246153691 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733246153691Cleaning up temporary data from old regions at 1733246153858 (+167 ms)Region opened successfully at 1733246153884 (+26 ms) 2024-12-03T17:15:53,886 INFO [master/27e0f852a164:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-03T17:15:53,930 DEBUG [master/27e0f852a164:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72f95140, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=27e0f852a164/172.17.0.3:0 2024-12-03T17:15:53,972 INFO [master/27e0f852a164:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-03T17:15:53,985 INFO [master/27e0f852a164:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-03T17:15:53,985 INFO [master/27e0f852a164:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-03T17:15:53,989 INFO [master/27e0f852a164:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T17:15:53,990 INFO [master/27e0f852a164:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-03T17:15:53,997 INFO [master/27e0f852a164:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-12-03T17:15:53,997 INFO [master/27e0f852a164:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-03T17:15:54,032 INFO [master/27e0f852a164:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-03T17:15:54,046 DEBUG [master/27e0f852a164:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38523-0x1010d48f3a20000, quorum=127.0.0.1:56976, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-03T17:15:54,049 DEBUG [master/27e0f852a164:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-03T17:15:54,053 INFO [master/27e0f852a164:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-03T17:15:54,056 DEBUG [master/27e0f852a164:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38523-0x1010d48f3a20000, quorum=127.0.0.1:56976, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-03T17:15:54,059 DEBUG [master/27e0f852a164:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-03T17:15:54,061 INFO [master/27e0f852a164:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-03T17:15:54,072 DEBUG [master/27e0f852a164:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38523-0x1010d48f3a20000, quorum=127.0.0.1:56976, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-03T17:15:54,076 DEBUG [master/27e0f852a164:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-03T17:15:54,083 DEBUG [master/27e0f852a164:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38523-0x1010d48f3a20000, quorum=127.0.0.1:56976, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-03T17:15:54,085 DEBUG [master/27e0f852a164:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-03T17:15:54,110 DEBUG [master/27e0f852a164:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38523-0x1010d48f3a20000, quorum=127.0.0.1:56976, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-03T17:15:54,117 DEBUG [master/27e0f852a164:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-03T17:15:54,123 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41167-0x1010d48f3a20001, quorum=127.0.0.1:56976, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T17:15:54,123 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41167-0x1010d48f3a20001, quorum=127.0.0.1:56976, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T17:15:54,123 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38523-0x1010d48f3a20000, quorum=127.0.0.1:56976, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T17:15:54,123 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38523-0x1010d48f3a20000, quorum=127.0.0.1:56976, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T17:15:54,126 INFO [master/27e0f852a164:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=27e0f852a164,38523,1733246150595, sessionid=0x1010d48f3a20000, setting cluster-up flag (Was=false) 2024-12-03T17:15:54,148 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41167-0x1010d48f3a20001, quorum=127.0.0.1:56976, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T17:15:54,148 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38523-0x1010d48f3a20000, quorum=127.0.0.1:56976, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T17:15:54,158 DEBUG [master/27e0f852a164:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-03T17:15:54,161 DEBUG [master/27e0f852a164:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=27e0f852a164,38523,1733246150595 2024-12-03T17:15:54,168 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38523-0x1010d48f3a20000, quorum=127.0.0.1:56976, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T17:15:54,168 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41167-0x1010d48f3a20001, quorum=127.0.0.1:56976, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T17:15:54,176 DEBUG [master/27e0f852a164:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-03T17:15:54,178 DEBUG [master/27e0f852a164:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=27e0f852a164,38523,1733246150595 2024-12-03T17:15:54,186 INFO [master/27e0f852a164:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-03T17:15:54,256 INFO [RS:0;27e0f852a164:41167 {}] regionserver.HRegionServer(746): ClusterId : ab945027-d9b6-4a24-a77e-0e4db28ff7fe 2024-12-03T17:15:54,260 DEBUG [RS:0;27e0f852a164:41167 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-03T17:15:54,267 DEBUG [RS:0;27e0f852a164:41167 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-03T17:15:54,267 DEBUG [RS:0;27e0f852a164:41167 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-03T17:15:54,271 DEBUG [RS:0;27e0f852a164:41167 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-03T17:15:54,272 DEBUG [RS:0;27e0f852a164:41167 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b3f04f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=27e0f852a164/172.17.0.3:0 2024-12-03T17:15:54,290 DEBUG [RS:0;27e0f852a164:41167 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;27e0f852a164:41167 2024-12-03T17:15:54,295 INFO [RS:0;27e0f852a164:41167 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-03T17:15:54,295 INFO [RS:0;27e0f852a164:41167 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-03T17:15:54,296 DEBUG [RS:0;27e0f852a164:41167 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-03T17:15:54,296 DEBUG [master/27e0f852a164:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T17:15:54,300 INFO [RS:0;27e0f852a164:41167 {}] regionserver.HRegionServer(2659): reportForDuty to master=27e0f852a164,38523,1733246150595 with port=41167, startcode=1733246151593 2024-12-03T17:15:54,310 INFO [master/27e0f852a164:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-03T17:15:54,318 DEBUG [RS:0;27e0f852a164:41167 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-03T17:15:54,319 INFO [master/27e0f852a164:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-03T17:15:54,326 DEBUG [master/27e0f852a164:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 27e0f852a164,38523,1733246150595 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-03T17:15:54,334 DEBUG [master/27e0f852a164:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/27e0f852a164:0, corePoolSize=5, maxPoolSize=5 2024-12-03T17:15:54,335 DEBUG [master/27e0f852a164:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/27e0f852a164:0, corePoolSize=5, maxPoolSize=5 2024-12-03T17:15:54,335 DEBUG [master/27e0f852a164:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/27e0f852a164:0, corePoolSize=5, maxPoolSize=5 2024-12-03T17:15:54,335 DEBUG [master/27e0f852a164:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/27e0f852a164:0, corePoolSize=5, maxPoolSize=5 2024-12-03T17:15:54,335 DEBUG [master/27e0f852a164:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/27e0f852a164:0, corePoolSize=10, maxPoolSize=10 2024-12-03T17:15:54,336 DEBUG [master/27e0f852a164:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/27e0f852a164:0, corePoolSize=1, maxPoolSize=1 2024-12-03T17:15:54,336 DEBUG [master/27e0f852a164:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/27e0f852a164:0, corePoolSize=2, maxPoolSize=2 2024-12-03T17:15:54,336 DEBUG [master/27e0f852a164:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/27e0f852a164:0, corePoolSize=1, maxPoolSize=1 2024-12-03T17:15:54,359 INFO [master/27e0f852a164:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733246184359 2024-12-03T17:15:54,361 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T17:15:54,361 INFO [master/27e0f852a164:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-03T17:15:54,362 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-03T17:15:54,363 INFO [master/27e0f852a164:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-03T17:15:54,370 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34139, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-03T17:15:54,371 INFO [master/27e0f852a164:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-03T17:15:54,372 INFO [master/27e0f852a164:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-03T17:15:54,372 INFO [master/27e0f852a164:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-03T17:15:54,373 INFO [master/27e0f852a164:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-03T17:15:54,376 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:15:54,374 INFO [master/27e0f852a164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-03T17:15:54,376 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-03T17:15:54,381 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38523 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:15:54,383 INFO [master/27e0f852a164:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-03T17:15:54,385 INFO [master/27e0f852a164:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-03T17:15:54,385 INFO [master/27e0f852a164:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-03T17:15:54,391 INFO [master/27e0f852a164:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-03T17:15:54,391 INFO [master/27e0f852a164:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-03T17:15:54,395 DEBUG [master/27e0f852a164:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/27e0f852a164:0:becomeActiveMaster-HFileCleaner.large.0-1733246154393,5,FailOnTimeoutGroup] 2024-12-03T17:15:54,396 DEBUG [master/27e0f852a164:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/27e0f852a164:0:becomeActiveMaster-HFileCleaner.small.0-1733246154395,5,FailOnTimeoutGroup] 2024-12-03T17:15:54,396 INFO [master/27e0f852a164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-03T17:15:54,396 INFO [master/27e0f852a164:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-03T17:15:54,398 INFO [master/27e0f852a164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-03T17:15:54,398 INFO [master/27e0f852a164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-03T17:15:54,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741831_1007 (size=1321) 2024-12-03T17:15:54,427 DEBUG [RS:0;27e0f852a164:41167 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-03T17:15:54,427 WARN [RS:0;27e0f852a164:41167 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-03T17:15:54,529 INFO [RS:0;27e0f852a164:41167 {}] regionserver.HRegionServer(2659): reportForDuty to master=27e0f852a164,38523,1733246150595 with port=41167, startcode=1733246151593 2024-12-03T17:15:54,533 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38523 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 27e0f852a164,41167,1733246151593 2024-12-03T17:15:54,538 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38523 {}] master.ServerManager(517): Registering regionserver=27e0f852a164,41167,1733246151593 2024-12-03T17:15:54,560 DEBUG [RS:0;27e0f852a164:41167 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4 2024-12-03T17:15:54,560 DEBUG [RS:0;27e0f852a164:41167 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:39175 2024-12-03T17:15:54,560 DEBUG [RS:0;27e0f852a164:41167 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-03T17:15:54,569 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38523-0x1010d48f3a20000, quorum=127.0.0.1:56976, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T17:15:54,580 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [27e0f852a164,41167,1733246151593] 2024-12-03T17:15:54,583 DEBUG [RS:0;27e0f852a164:41167 {}] zookeeper.ZKUtil(111): regionserver:41167-0x1010d48f3a20001, quorum=127.0.0.1:56976, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/27e0f852a164,41167,1733246151593 2024-12-03T17:15:54,583 WARN [RS:0;27e0f852a164:41167 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T17:15:54,585 INFO [RS:0;27e0f852a164:41167 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T17:15:54,589 DEBUG [RS:0;27e0f852a164:41167 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/WALs/27e0f852a164,41167,1733246151593 2024-12-03T17:15:54,672 INFO [RS:0;27e0f852a164:41167 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-03T17:15:54,702 INFO [RS:0;27e0f852a164:41167 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-03T17:15:54,721 INFO [RS:0;27e0f852a164:41167 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-03T17:15:54,722 INFO [RS:0;27e0f852a164:41167 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T17:15:54,723 INFO [RS:0;27e0f852a164:41167 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-03T17:15:54,732 INFO [RS:0;27e0f852a164:41167 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-03T17:15:54,735 INFO [RS:0;27e0f852a164:41167 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-03T17:15:54,736 DEBUG [RS:0;27e0f852a164:41167 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/27e0f852a164:0, corePoolSize=1, maxPoolSize=1 2024-12-03T17:15:54,736 DEBUG [RS:0;27e0f852a164:41167 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/27e0f852a164:0, corePoolSize=1, maxPoolSize=1 2024-12-03T17:15:54,736 DEBUG [RS:0;27e0f852a164:41167 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/27e0f852a164:0, corePoolSize=1, maxPoolSize=1 2024-12-03T17:15:54,736 DEBUG [RS:0;27e0f852a164:41167 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/27e0f852a164:0, corePoolSize=1, maxPoolSize=1 2024-12-03T17:15:54,736 DEBUG [RS:0;27e0f852a164:41167 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/27e0f852a164:0, corePoolSize=1, maxPoolSize=1 2024-12-03T17:15:54,736 DEBUG [RS:0;27e0f852a164:41167 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/27e0f852a164:0, corePoolSize=2, maxPoolSize=2 2024-12-03T17:15:54,736 DEBUG [RS:0;27e0f852a164:41167 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/27e0f852a164:0, corePoolSize=1, maxPoolSize=1 2024-12-03T17:15:54,737 DEBUG [RS:0;27e0f852a164:41167 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/27e0f852a164:0, corePoolSize=1, maxPoolSize=1 2024-12-03T17:15:54,737 DEBUG [RS:0;27e0f852a164:41167 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/27e0f852a164:0, corePoolSize=1, maxPoolSize=1 2024-12-03T17:15:54,737 DEBUG [RS:0;27e0f852a164:41167 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/27e0f852a164:0, corePoolSize=1, maxPoolSize=1 2024-12-03T17:15:54,737 DEBUG [RS:0;27e0f852a164:41167 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/27e0f852a164:0, corePoolSize=1, maxPoolSize=1 2024-12-03T17:15:54,737 DEBUG [RS:0;27e0f852a164:41167 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/27e0f852a164:0, corePoolSize=1, maxPoolSize=1 2024-12-03T17:15:54,737 DEBUG [RS:0;27e0f852a164:41167 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/27e0f852a164:0, corePoolSize=3, maxPoolSize=3 2024-12-03T17:15:54,738 DEBUG [RS:0;27e0f852a164:41167 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0, corePoolSize=3, maxPoolSize=3 2024-12-03T17:15:54,743 INFO [RS:0;27e0f852a164:41167 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T17:15:54,743 INFO [RS:0;27e0f852a164:41167 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T17:15:54,743 INFO [RS:0;27e0f852a164:41167 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T17:15:54,744 INFO [RS:0;27e0f852a164:41167 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-03T17:15:54,744 INFO [RS:0;27e0f852a164:41167 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-03T17:15:54,744 INFO [RS:0;27e0f852a164:41167 {}] hbase.ChoreService(168): Chore ScheduledChore name=27e0f852a164,41167,1733246151593-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T17:15:54,777 INFO [RS:0;27e0f852a164:41167 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-03T17:15:54,780 INFO [RS:0;27e0f852a164:41167 {}] hbase.ChoreService(168): Chore ScheduledChore name=27e0f852a164,41167,1733246151593-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T17:15:54,781 INFO [RS:0;27e0f852a164:41167 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T17:15:54,782 INFO [RS:0;27e0f852a164:41167 {}] regionserver.Replication(171): 27e0f852a164,41167,1733246151593 started 2024-12-03T17:15:54,814 INFO [RS:0;27e0f852a164:41167 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T17:15:54,815 INFO [RS:0;27e0f852a164:41167 {}] regionserver.HRegionServer(1482): Serving as 27e0f852a164,41167,1733246151593, RpcServer on 27e0f852a164/172.17.0.3:41167, sessionid=0x1010d48f3a20001 2024-12-03T17:15:54,816 DEBUG [RS:0;27e0f852a164:41167 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-03T17:15:54,817 DEBUG [RS:0;27e0f852a164:41167 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 27e0f852a164,41167,1733246151593 2024-12-03T17:15:54,817 DEBUG [RS:0;27e0f852a164:41167 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '27e0f852a164,41167,1733246151593' 2024-12-03T17:15:54,817 DEBUG [RS:0;27e0f852a164:41167 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-03T17:15:54,818 DEBUG [RS:0;27e0f852a164:41167 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-03T17:15:54,821 DEBUG [RS:0;27e0f852a164:41167 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-03T17:15:54,822 DEBUG [RS:0;27e0f852a164:41167 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-03T17:15:54,822 DEBUG [RS:0;27e0f852a164:41167 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 27e0f852a164,41167,1733246151593 2024-12-03T17:15:54,822 DEBUG [RS:0;27e0f852a164:41167 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '27e0f852a164,41167,1733246151593' 2024-12-03T17:15:54,822 DEBUG [RS:0;27e0f852a164:41167 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-03T17:15:54,824 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-03T17:15:54,824 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4 2024-12-03T17:15:54,828 DEBUG [RS:0;27e0f852a164:41167 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-03T17:15:54,831 DEBUG [RS:0;27e0f852a164:41167 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-03T17:15:54,831 INFO [RS:0;27e0f852a164:41167 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-03T17:15:54,831 INFO [RS:0;27e0f852a164:41167 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-03T17:15:54,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741832_1008 (size=32) 2024-12-03T17:15:54,939 INFO [RS:0;27e0f852a164:41167 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T17:15:54,945 INFO [RS:0;27e0f852a164:41167 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=27e0f852a164%2C41167%2C1733246151593, suffix=, logDir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/WALs/27e0f852a164,41167,1733246151593, archiveDir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/oldWALs, maxLogs=32 2024-12-03T17:15:54,968 DEBUG [RS:0;27e0f852a164:41167 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/WALs/27e0f852a164,41167,1733246151593/27e0f852a164%2C41167%2C1733246151593.1733246154949, exclude list is [], retry=0 2024-12-03T17:15:54,975 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38185,DS-1f8e4f8f-fe3f-41b1-8213-7117a2e64c04,DISK] 2024-12-03T17:15:54,983 INFO [RS:0;27e0f852a164:41167 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/WALs/27e0f852a164,41167,1733246151593/27e0f852a164%2C41167%2C1733246151593.1733246154949 2024-12-03T17:15:54,984 DEBUG [RS:0;27e0f852a164:41167 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39969:39969)] 2024-12-03T17:15:55,254 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T17:15:55,265 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-03T17:15:55,273 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-03T17:15:55,273 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:15:55,274 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T17:15:55,275 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-03T17:15:55,279 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-03T17:15:55,279 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:15:55,280 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T17:15:55,281 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-03T17:15:55,285 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-03T17:15:55,285 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:15:55,290 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T17:15:55,290 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-03T17:15:55,295 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-03T17:15:55,295 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:15:55,297 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T17:15:55,297 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-03T17:15:55,299 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/hbase/meta/1588230740 2024-12-03T17:15:55,301 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/hbase/meta/1588230740 2024-12-03T17:15:55,307 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-03T17:15:55,308 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-03T17:15:55,310 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T17:15:55,314 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-03T17:15:55,321 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T17:15:55,323 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65060177, jitterRate=-0.030527815222740173}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T17:15:55,326 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733246155254Initializing all the Stores at 1733246155264 (+10 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733246155264Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733246155265 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733246155265Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733246155265Cleaning up temporary data from old regions at 1733246155308 (+43 ms)Region opened successfully at 1733246155325 (+17 ms) 2024-12-03T17:15:55,326 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-03T17:15:55,326 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-03T17:15:55,327 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-03T17:15:55,327 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-03T17:15:55,327 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-03T17:15:55,329 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-03T17:15:55,329 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733246155326Disabling compacts and flushes for region at 1733246155326Disabling writes for close at 1733246155327 (+1 ms)Writing region close event to WAL at 1733246155328 (+1 ms)Closed at 1733246155329 (+1 ms) 2024-12-03T17:15:55,334 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T17:15:55,334 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-03T17:15:55,345 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-03T17:15:55,363 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T17:15:55,372 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-03T17:15:55,526 DEBUG [27e0f852a164:38523 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-03T17:15:55,545 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=27e0f852a164,41167,1733246151593 2024-12-03T17:15:55,559 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 27e0f852a164,41167,1733246151593, state=OPENING 2024-12-03T17:15:55,569 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-03T17:15:55,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41167-0x1010d48f3a20001, quorum=127.0.0.1:56976, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T17:15:55,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38523-0x1010d48f3a20000, quorum=127.0.0.1:56976, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T17:15:55,580 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T17:15:55,582 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T17:15:55,583 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T17:15:55,586 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=27e0f852a164,41167,1733246151593}] 2024-12-03T17:15:55,777 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-03T17:15:55,787 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48273, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-03T17:15:55,822 INFO [RS_OPEN_META-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-03T17:15:55,823 INFO [RS_OPEN_META-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T17:15:55,827 INFO [RS_OPEN_META-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-03T17:15:55,842 INFO [RS_OPEN_META-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=27e0f852a164%2C41167%2C1733246151593.meta, suffix=.meta, logDir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/WALs/27e0f852a164,41167,1733246151593, archiveDir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/oldWALs, maxLogs=32 2024-12-03T17:15:55,867 DEBUG [RS_OPEN_META-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/WALs/27e0f852a164,41167,1733246151593/27e0f852a164%2C41167%2C1733246151593.meta.1733246155844.meta, exclude list is [], retry=0 2024-12-03T17:15:55,875 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38185,DS-1f8e4f8f-fe3f-41b1-8213-7117a2e64c04,DISK] 2024-12-03T17:15:55,881 INFO [RS_OPEN_META-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/WALs/27e0f852a164,41167,1733246151593/27e0f852a164%2C41167%2C1733246151593.meta.1733246155844.meta 2024-12-03T17:15:55,882 DEBUG [RS_OPEN_META-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39969:39969)] 2024-12-03T17:15:55,882 DEBUG [RS_OPEN_META-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-03T17:15:55,885 DEBUG [RS_OPEN_META-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-03T17:15:55,890 DEBUG [RS_OPEN_META-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-03T17:15:55,898 INFO [RS_OPEN_META-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-03T17:15:55,905 DEBUG [RS_OPEN_META-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-03T17:15:55,906 DEBUG [RS_OPEN_META-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T17:15:55,906 DEBUG [RS_OPEN_META-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-03T17:15:55,906 DEBUG [RS_OPEN_META-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-03T17:15:55,916 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-03T17:15:55,919 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-03T17:15:55,919 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:15:55,923 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T17:15:55,923 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-03T17:15:55,926 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-03T17:15:55,926 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:15:55,932 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T17:15:55,932 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-03T17:15:55,936 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-03T17:15:55,936 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:15:55,937 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T17:15:55,938 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-03T17:15:55,940 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-03T17:15:55,941 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:15:55,942 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T17:15:55,943 DEBUG [RS_OPEN_META-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-03T17:15:55,948 DEBUG [RS_OPEN_META-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/hbase/meta/1588230740 2024-12-03T17:15:55,953 DEBUG [RS_OPEN_META-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/hbase/meta/1588230740 2024-12-03T17:15:55,958 DEBUG [RS_OPEN_META-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-03T17:15:55,958 DEBUG [RS_OPEN_META-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-03T17:15:55,960 DEBUG [RS_OPEN_META-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T17:15:55,964 DEBUG [RS_OPEN_META-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-03T17:15:55,966 INFO [RS_OPEN_META-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64649303, jitterRate=-0.036650314927101135}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T17:15:55,967 DEBUG [RS_OPEN_META-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-03T17:15:55,970 DEBUG [RS_OPEN_META-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733246155907Writing region info on filesystem at 1733246155908 (+1 ms)Initializing all the Stores at 1733246155913 (+5 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733246155913Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733246155915 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733246155915Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733246155915Cleaning up temporary data from old regions at 1733246155958 (+43 ms)Running coprocessor post-open hooks at 1733246155967 (+9 ms)Region opened successfully at 1733246155969 (+2 ms) 2024-12-03T17:15:55,979 INFO [RS_OPEN_META-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733246155763 2024-12-03T17:15:56,000 DEBUG [RS_OPEN_META-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-03T17:15:56,001 INFO [RS_OPEN_META-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-03T17:15:56,003 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=27e0f852a164,41167,1733246151593 2024-12-03T17:15:56,006 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 27e0f852a164,41167,1733246151593, state=OPEN 2024-12-03T17:15:56,025 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38523-0x1010d48f3a20000, quorum=127.0.0.1:56976, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T17:15:56,025 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T17:15:56,025 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=27e0f852a164,41167,1733246151593 2024-12-03T17:15:56,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41167-0x1010d48f3a20001, quorum=127.0.0.1:56976, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T17:15:56,027 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T17:15:56,035 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-03T17:15:56,035 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=27e0f852a164,41167,1733246151593 in 441 msec 2024-12-03T17:15:56,046 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-03T17:15:56,046 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 693 msec 2024-12-03T17:15:56,049 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T17:15:56,049 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-03T17:15:56,084 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:15:56,087 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:15:56,118 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:15:56,121 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54597, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:15:56,154 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.9170 sec 2024-12-03T17:15:56,159 INFO [master/27e0f852a164:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733246156158, completionTime=-1 2024-12-03T17:15:56,163 INFO [master/27e0f852a164:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-03T17:15:56,163 DEBUG [master/27e0f852a164:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T17:15:56,251 INFO [master/27e0f852a164:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-03T17:15:56,251 INFO [master/27e0f852a164:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733246216251 2024-12-03T17:15:56,251 INFO [master/27e0f852a164:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733246276251 2024-12-03T17:15:56,251 INFO [master/27e0f852a164:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 87 msec 2024-12-03T17:15:56,255 INFO [master/27e0f852a164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=27e0f852a164,38523,1733246150595-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T17:15:56,255 INFO [master/27e0f852a164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=27e0f852a164,38523,1733246150595-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T17:15:56,256 INFO [master/27e0f852a164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=27e0f852a164,38523,1733246150595-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T17:15:56,268 INFO [master/27e0f852a164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-27e0f852a164:38523, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T17:15:56,269 INFO [master/27e0f852a164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-03T17:15:56,278 INFO [master/27e0f852a164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-03T17:15:56,285 DEBUG [master/27e0f852a164:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-03T17:15:56,327 INFO [master/27e0f852a164:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.513sec 2024-12-03T17:15:56,329 INFO [master/27e0f852a164:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-03T17:15:56,331 INFO [master/27e0f852a164:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-03T17:15:56,332 INFO [master/27e0f852a164:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-03T17:15:56,333 INFO [master/27e0f852a164:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-03T17:15:56,333 INFO [master/27e0f852a164:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-03T17:15:56,335 INFO [master/27e0f852a164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=27e0f852a164,38523,1733246150595-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T17:15:56,335 INFO [master/27e0f852a164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=27e0f852a164,38523,1733246150595-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-03T17:15:56,365 DEBUG [master/27e0f852a164:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-03T17:15:56,366 INFO [master/27e0f852a164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-03T17:15:56,367 INFO [master/27e0f852a164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=27e0f852a164,38523,1733246150595-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T17:15:56,377 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45544f78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:15:56,381 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-03T17:15:56,381 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-03T17:15:56,394 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:15:56,398 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:15:56,415 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:15:56,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:15:56,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:15:56,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b5c7a62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:15:56,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:15:56,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:15:56,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:15:56,430 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45498, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:15:56,436 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2de67725, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:15:56,436 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:15:56,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:15:56,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:15:56,454 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51712, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:15:56,492 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=27e0f852a164,38523,1733246150595 2024-12-03T17:15:56,520 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=210, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=1114, ProcessCount=11, AvailableMemoryMB=6385 2024-12-03T17:15:56,533 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T17:15:56,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 27e0f852a164,38523,1733246150595 2024-12-03T17:15:56,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5b65baa9 2024-12-03T17:15:56,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T17:15:56,559 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45508, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T17:15:56,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T17:15:56,569 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T17:15:56,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-03T17:15:56,593 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T17:15:56,597 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:15:56,602 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T17:15:56,604 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-12-03T17:15:56,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T17:15:56,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741835_1011 (size=1061) 2024-12-03T17:15:56,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T17:15:56,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T17:15:57,057 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4 2024-12-03T17:15:57,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741836_1012 (size=53) 2024-12-03T17:15:57,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T17:15:57,497 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T17:15:57,498 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing e1e0418f3d66f886e020279128ac6f55, disabling compactions & flushes 2024-12-03T17:15:57,498 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:15:57,498 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:15:57,498 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. after waiting 0 ms 2024-12-03T17:15:57,498 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:15:57,498 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:15:57,498 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for e1e0418f3d66f886e020279128ac6f55: Waiting for close lock at 1733246157497Disabling compacts and flushes for region at 1733246157497Disabling writes for close at 1733246157498 (+1 ms)Writing region close event to WAL at 1733246157498Closed at 1733246157498 2024-12-03T17:15:57,502 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T17:15:57,509 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733246157502"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733246157502"}]},"ts":"1733246157502"} 2024-12-03T17:15:57,518 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T17:15:57,521 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T17:15:57,525 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733246157521"}]},"ts":"1733246157521"} 2024-12-03T17:15:57,533 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-03T17:15:57,535 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e1e0418f3d66f886e020279128ac6f55, ASSIGN}] 2024-12-03T17:15:57,541 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e1e0418f3d66f886e020279128ac6f55, ASSIGN 2024-12-03T17:15:57,544 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e1e0418f3d66f886e020279128ac6f55, ASSIGN; state=OFFLINE, location=27e0f852a164,41167,1733246151593; forceNewPlan=false, retain=false 2024-12-03T17:15:57,698 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e1e0418f3d66f886e020279128ac6f55, regionState=OPENING, regionLocation=27e0f852a164,41167,1733246151593 2024-12-03T17:15:57,715 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e1e0418f3d66f886e020279128ac6f55, ASSIGN because future has completed 2024-12-03T17:15:57,727 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593}] 2024-12-03T17:15:57,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T17:15:57,925 INFO [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:15:57,925 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} 2024-12-03T17:15:57,926 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e1e0418f3d66f886e020279128ac6f55 2024-12-03T17:15:57,927 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T17:15:57,927 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for e1e0418f3d66f886e020279128ac6f55 2024-12-03T17:15:57,927 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for e1e0418f3d66f886e020279128ac6f55 2024-12-03T17:15:57,952 INFO [StoreOpener-e1e0418f3d66f886e020279128ac6f55-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e1e0418f3d66f886e020279128ac6f55 2024-12-03T17:15:57,967 INFO [StoreOpener-e1e0418f3d66f886e020279128ac6f55-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T17:15:57,968 INFO [StoreOpener-e1e0418f3d66f886e020279128ac6f55-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e1e0418f3d66f886e020279128ac6f55 columnFamilyName A 2024-12-03T17:15:57,968 DEBUG [StoreOpener-e1e0418f3d66f886e020279128ac6f55-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:15:57,979 INFO [StoreOpener-e1e0418f3d66f886e020279128ac6f55-1 {}] regionserver.HStore(327): Store=e1e0418f3d66f886e020279128ac6f55/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T17:15:57,979 INFO [StoreOpener-e1e0418f3d66f886e020279128ac6f55-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e1e0418f3d66f886e020279128ac6f55 2024-12-03T17:15:57,995 INFO [StoreOpener-e1e0418f3d66f886e020279128ac6f55-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T17:15:57,996 INFO [StoreOpener-e1e0418f3d66f886e020279128ac6f55-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e1e0418f3d66f886e020279128ac6f55 columnFamilyName B 2024-12-03T17:15:58,000 DEBUG [StoreOpener-e1e0418f3d66f886e020279128ac6f55-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:15:58,004 INFO [StoreOpener-e1e0418f3d66f886e020279128ac6f55-1 {}] regionserver.HStore(327): Store=e1e0418f3d66f886e020279128ac6f55/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T17:15:58,004 INFO [StoreOpener-e1e0418f3d66f886e020279128ac6f55-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e1e0418f3d66f886e020279128ac6f55 2024-12-03T17:15:58,017 INFO [StoreOpener-e1e0418f3d66f886e020279128ac6f55-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T17:15:58,017 INFO [StoreOpener-e1e0418f3d66f886e020279128ac6f55-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e1e0418f3d66f886e020279128ac6f55 columnFamilyName C 2024-12-03T17:15:58,018 DEBUG [StoreOpener-e1e0418f3d66f886e020279128ac6f55-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:15:58,019 INFO [StoreOpener-e1e0418f3d66f886e020279128ac6f55-1 {}] regionserver.HStore(327): Store=e1e0418f3d66f886e020279128ac6f55/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T17:15:58,020 INFO [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:15:58,020 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for e1e0418f3d66f886e020279128ac6f55 2024-12-03T17:15:58,022 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55 2024-12-03T17:15:58,024 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55 2024-12-03T17:15:58,039 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for e1e0418f3d66f886e020279128ac6f55 2024-12-03T17:15:58,039 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for e1e0418f3d66f886e020279128ac6f55 2024-12-03T17:15:58,042 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T17:15:58,054 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for e1e0418f3d66f886e020279128ac6f55 2024-12-03T17:15:58,094 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T17:15:58,106 INFO [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened e1e0418f3d66f886e020279128ac6f55; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74695279, jitterRate=0.11304639279842377}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T17:15:58,107 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e1e0418f3d66f886e020279128ac6f55 2024-12-03T17:15:58,108 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for e1e0418f3d66f886e020279128ac6f55: Running coprocessor pre-open hook at 1733246157927Writing region info on filesystem at 1733246157927Initializing all the Stores at 1733246157945 (+18 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733246157945Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733246157951 (+6 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733246157951Cleaning up temporary data from old regions at 1733246158040 (+89 ms)Running coprocessor post-open hooks at 1733246158107 (+67 ms)Region opened successfully at 1733246158108 (+1 ms) 2024-12-03T17:15:58,121 INFO [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., pid=6, masterSystemTime=1733246157896 2024-12-03T17:15:58,136 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:15:58,136 INFO [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:15:58,143 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e1e0418f3d66f886e020279128ac6f55, regionState=OPEN, openSeqNum=2, regionLocation=27e0f852a164,41167,1733246151593 2024-12-03T17:15:58,154 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 because future has completed 2024-12-03T17:15:58,184 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-03T17:15:58,187 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 in 439 msec 2024-12-03T17:15:58,199 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-03T17:15:58,199 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e1e0418f3d66f886e020279128ac6f55, ASSIGN in 648 msec 2024-12-03T17:15:58,207 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T17:15:58,208 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733246158207"}]},"ts":"1733246158207"} 2024-12-03T17:15:58,233 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-03T17:15:58,238 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T17:15:58,256 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.6640 sec 2024-12-03T17:15:58,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T17:15:58,786 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:15:58,794 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:15:58,794 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:15:58,794 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:15:58,796 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:15:58,797 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:15:58,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:15:58,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9f75e9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:15:58,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:15:58,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:15:58,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:15:58,809 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55776, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:15:58,813 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dcb8366, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:15:58,814 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:15:58,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:15:58,817 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:15:58,821 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47728, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:15:58,827 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T17:15:58,832 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 27e0f852a164,38523,1733246150595 2024-12-03T17:15:58,832 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@545a7bf9 2024-12-03T17:15:58,837 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T17:15:58,847 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55780, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T17:15:58,870 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:15:58,870 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:15:58,870 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:15:58,879 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:15:58,880 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:15:58,880 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:15:58,883 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48ffac79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:15:58,883 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:15:58,884 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:15:58,890 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:15:58,896 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55802, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:15:58,900 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d0184e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:15:58,903 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:15:58,904 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:15:58,904 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:15:58,909 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:15:58,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:15:58,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:15:58,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cbf42c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:15:58,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:15:58,911 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:15:58,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:15:58,923 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55828, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:15:58,934 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59cb0f4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:15:58,938 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:15:58,938 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:15:58,939 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:15:58,947 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:15:58,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:15:58,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:15:58,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@603c95bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:15:58,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:15:58,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:15:58,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:15:58,963 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55838, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:15:58,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b6b9e8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:15:58,974 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:15:58,974 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:15:58,974 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:15:58,985 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:15:58,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:15:58,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:15:58,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50c9e84c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:15:58,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:15:58,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:15:58,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:15:59,000 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55852, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:15:59,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60b8091e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:15:59,005 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:15:59,005 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:15:59,006 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:15:59,008 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:15:59,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:15:59,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:15:59,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5805ae29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:15:59,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:15:59,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:15:59,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:15:59,015 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55874, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:15:59,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54c50a81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:15:59,036 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ff89310, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:15:59,036 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:15:59,037 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:15:59,039 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:15:59,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:15:59,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:15:59,041 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:15:59,042 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4454ce2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:15:59,042 DEBUG [Time-limited test {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:15:59,042 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:15:59,045 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55898, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:15:59,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d37adaa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:15:59,051 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74a9c1c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:15:59,051 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:15:59,051 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:15:59,055 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:15:59,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:15:59,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:15:59,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5be3f202, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:15:59,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:15:59,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:15:59,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:15:59,059 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55920, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:15:59,062 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@739049a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:15:59,068 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:15:59,068 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:15:59,069 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:15:59,070 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:15:59,071 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:15:59,071 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:15:59,072 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e3e6f45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:15:59,072 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:15:59,072 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:15:59,073 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:15:59,075 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55940, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:15:59,076 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13f190d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:15:59,079 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cdd76a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:15:59,079 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:15:59,079 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:15:59,081 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:15:59,082 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:15:59,082 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:15:59,082 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56900a9b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:15:59,082 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:15:59,083 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:15:59,083 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:15:59,086 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55962, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:15:59,092 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51ceab7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:15:59,107 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:15:59,107 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:15:59,107 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:15:59,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:15:59,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:15:59,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:15:59,111 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:15:59,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:15:59,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:15:59,113 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47738, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:15:59,118 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47746, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:15:59,118 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47748, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:15:59,121 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:15:59,128 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2] 2024-12-03T17:15:59,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2] 2024-12-03T17:15:59,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2] 2024-12-03T17:15:59,131 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:15:59,131 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:15:59,132 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:15:59,134 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T17:15:59,136 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:15:59,136 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:15:59,139 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47750, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:15:59,139 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47752, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:15:59,141 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:15:59,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2] 2024-12-03T17:15:59,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2] 2024-12-03T17:15:59,151 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:15:59,151 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:15:59,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:15:59,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:15:59,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:15:59,156 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47760, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:15:59,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-12-03T17:15:59,159 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:15:59,159 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:15:59,159 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47768, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:15:59,161 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:15:59,162 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:15:59,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:15:59,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:15:59,165 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47782, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:15:59,168 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47788, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:15:59,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2] 2024-12-03T17:15:59,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2] 2024-12-03T17:15:59,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T17:15:59,178 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T17:15:59,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2] 2024-12-03T17:15:59,187 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T17:15:59,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2] 2024-12-03T17:15:59,189 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T17:15:59,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on e1e0418f3d66f886e020279128ac6f55 2024-12-03T17:15:59,274 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e1e0418f3d66f886e020279128ac6f55 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T17:15:59,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T17:15:59,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=A 2024-12-03T17:15:59,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:15:59,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=B 2024-12-03T17:15:59,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:15:59,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=C 2024-12-03T17:15:59,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:15:59,419 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T17:15:59,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:15:59,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. as already flushing 2024-12-03T17:15:59,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:15:59,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:15:59,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:15:59,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:15:59,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/c65d6c13ed1942c6b9c430a51b7f0c40 is 50, key is test_row_0/A:col10/1733246159256/Put/seqid=0 2024-12-03T17:15:59,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T17:15:59,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741837_1013 (size=12001) 2024-12-03T17:15:59,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:15:59,581 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/c65d6c13ed1942c6b9c430a51b7f0c40 2024-12-03T17:15:59,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:15:59,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47748 deadline: 1733246219529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:15:59,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246219532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:15:59,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:15:59,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:15:59,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:15:59,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:15:59,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:15:59,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:15:59,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:15:59,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246219549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:15:59,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:15:59,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:15:59,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:15:59,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T17:15:59,622 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733246159490, max timestamp=1733246159571Num uniques 859; 2024-12-03T17:15:59,624 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T17:15:59,628 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733246159490, max timestamp=1733246159571Num uniques 859; 2024-12-03T17:15:59,629 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T17:15:59,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:15:59,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. as already flushing 2024-12-03T17:15:59,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:15:59,631 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:15:59,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:15:59,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:15:59,638 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733246159490, max timestamp=1733246159571Num uniques 859; 2024-12-03T17:15:59,639 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T17:15:59,736 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/a38c5bd6befd44b8b5e0591b4b5b0c83 is 50, key is test_row_0/B:col10/1733246159256/Put/seqid=0 2024-12-03T17:15:59,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741838_1014 (size=12001) 2024-12-03T17:15:59,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T17:15:59,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T17:15:59,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:15:59,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. as already flushing 2024-12-03T17:15:59,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:15:59,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:15:59,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:15:59,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:15:59,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:15:59,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246219791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:15:59,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:15:59,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:15:59,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:15:59,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:15:59,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:15:59,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47748 deadline: 1733246219814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:15:59,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246219818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:15:59,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:15:59,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:15:59,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:15:59,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:15:59,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:15:59,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:15:59,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:15:59,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246219822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:15:59,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:15:59,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:15:59,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:15:59,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:15:59,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1733246219829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:15:59,854 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:15:59,854 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:15:59,854 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:15:59,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:15:59,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246219937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:15:59,952 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:15:59,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:15:59,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:15:59,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:15:59,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47748 deadline: 1733246219956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:15:59,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:15:59,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246219956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:15:59,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:15:59,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:15:59,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:15:59,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:15:59,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:15:59,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:15:59,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:15:59,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246219957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:15:59,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:15:59,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:15:59,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:15:59,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:15:59,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T17:15:59,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1733246219964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:15:59,977 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:15:59,977 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:15:59,977 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:15:59,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:15:59,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. as already flushing 2024-12-03T17:15:59,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:15:59,979 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:15:59,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:15:59,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:00,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T17:16:00,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:00,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. as already flushing 2024-12-03T17:16:00,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:00,154 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:00,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:00,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:00,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:00,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246220167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:00,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:00,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:00,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:00,189 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/a38c5bd6befd44b8b5e0591b4b5b0c83 2024-12-03T17:16:00,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:00,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47748 deadline: 1733246220186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:00,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:00,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246220189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:00,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:00,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:00,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:00,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:00,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1733246220191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:00,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:00,198 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:00,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:00,198 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:00,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:00,198 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:00,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:00,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246220192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:00,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:00,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:00,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:00,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T17:16:00,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:00,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. as already flushing 2024-12-03T17:16:00,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:00,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:00,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T17:16:00,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:00,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:00,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/3077a4922e89443abbf0eca785510bf2 is 50, key is test_row_0/C:col10/1733246159256/Put/seqid=0 2024-12-03T17:16:00,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741839_1015 (size=12001) 2024-12-03T17:16:00,387 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/3077a4922e89443abbf0eca785510bf2 2024-12-03T17:16:00,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/c65d6c13ed1942c6b9c430a51b7f0c40 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/c65d6c13ed1942c6b9c430a51b7f0c40 2024-12-03T17:16:00,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T17:16:00,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:00,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. as already flushing 2024-12-03T17:16:00,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:00,492 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:00,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:00,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:00,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:00,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246220495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:00,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:00,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:00,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:00,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:00,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:00,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246220505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:00,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:00,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1733246220506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:00,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:00,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246220519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:00,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:00,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:00,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:00,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:00,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:00,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:00,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:00,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:00,533 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:00,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47748 deadline: 1733246220504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:00,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:00,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:00,543 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/c65d6c13ed1942c6b9c430a51b7f0c40, entries=150, sequenceid=13, filesize=11.7 K 2024-12-03T17:16:00,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:00,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/a38c5bd6befd44b8b5e0591b4b5b0c83 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/a38c5bd6befd44b8b5e0591b4b5b0c83 2024-12-03T17:16:00,659 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-03T17:16:00,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T17:16:00,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:00,671 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-03T17:16:00,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. as already flushing 2024-12-03T17:16:00,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:00,672 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:00,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:00,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:00,677 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/a38c5bd6befd44b8b5e0591b4b5b0c83, entries=150, sequenceid=13, filesize=11.7 K 2024-12-03T17:16:00,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/3077a4922e89443abbf0eca785510bf2 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/3077a4922e89443abbf0eca785510bf2 2024-12-03T17:16:00,732 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/3077a4922e89443abbf0eca785510bf2, entries=150, sequenceid=13, filesize=11.7 K 2024-12-03T17:16:00,760 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=127.47 KB/130530 for e1e0418f3d66f886e020279128ac6f55 in 1471ms, sequenceid=13, compaction requested=false 2024-12-03T17:16:00,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e1e0418f3d66f886e020279128ac6f55: 2024-12-03T17:16:00,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-03T17:16:00,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:00,841 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing e1e0418f3d66f886e020279128ac6f55 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-03T17:16:00,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=A 2024-12-03T17:16:00,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:00,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=B 2024-12-03T17:16:00,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:00,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=C 2024-12-03T17:16:00,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:00,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/914e83bbeaf14ece9ed23e921dbae1ca is 50, key is test_row_0/A:col10/1733246159815/Put/seqid=0 2024-12-03T17:16:00,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741840_1016 (size=12001) 2024-12-03T17:16:00,979 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/914e83bbeaf14ece9ed23e921dbae1ca 2024-12-03T17:16:01,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on e1e0418f3d66f886e020279128ac6f55 2024-12-03T17:16:01,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. as already flushing 2024-12-03T17:16:01,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/afc44c870ad044539c4a37a880dd36e1 is 50, key is test_row_0/B:col10/1733246159815/Put/seqid=0 2024-12-03T17:16:01,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741841_1017 (size=12001) 2024-12-03T17:16:01,135 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/afc44c870ad044539c4a37a880dd36e1 2024-12-03T17:16:01,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/1da0d6b3fff94ce08ed729056a9c4901 is 50, key is test_row_0/C:col10/1733246159815/Put/seqid=0 2024-12-03T17:16:01,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:01,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:01,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246221192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:01,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:01,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47748 deadline: 1733246221153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:01,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:01,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:01,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:01,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:01,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:01,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:01,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:01,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246221202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:01,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246221214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:01,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:01,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:01,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:01,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:01,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:01,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:01,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741842_1018 (size=12001) 2024-12-03T17:16:01,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:01,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1733246221233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:01,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:01,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:01,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:01,265 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/1da0d6b3fff94ce08ed729056a9c4901 2024-12-03T17:16:01,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/914e83bbeaf14ece9ed23e921dbae1ca as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/914e83bbeaf14ece9ed23e921dbae1ca 2024-12-03T17:16:01,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T17:16:01,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:01,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246221345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:01,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:01,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47748 deadline: 1733246221345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:01,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:01,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:01,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:01,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:01,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246221356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:01,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:01,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246221357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:01,395 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/914e83bbeaf14ece9ed23e921dbae1ca, entries=150, sequenceid=49, filesize=11.7 K 2024-12-03T17:16:01,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:01,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:01,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:01,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:01,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:01,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:01,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:01,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:01,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:01,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:01,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1733246221375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:01,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:01,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:01,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:01,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/afc44c870ad044539c4a37a880dd36e1 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/afc44c870ad044539c4a37a880dd36e1 2024-12-03T17:16:01,428 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T17:16:01,476 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/afc44c870ad044539c4a37a880dd36e1, entries=150, sequenceid=49, filesize=11.7 K 2024-12-03T17:16:01,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/1da0d6b3fff94ce08ed729056a9c4901 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/1da0d6b3fff94ce08ed729056a9c4901 2024-12-03T17:16:01,538 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/1da0d6b3fff94ce08ed729056a9c4901, entries=150, sequenceid=49, filesize=11.7 K 2024-12-03T17:16:01,539 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-03T17:16:01,539 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-03T17:16:01,542 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-03T17:16:01,542 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-03T17:16:01,543 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-03T17:16:01,543 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-03T17:16:01,544 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-03T17:16:01,544 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-03T17:16:01,551 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=60.38 KB/61830 for e1e0418f3d66f886e020279128ac6f55 in 710ms, sequenceid=49, compaction requested=false 2024-12-03T17:16:01,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for e1e0418f3d66f886e020279128ac6f55: 2024-12-03T17:16:01,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:01,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-12-03T17:16:01,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-12-03T17:16:01,577 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-03T17:16:01,577 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3750 sec 2024-12-03T17:16:01,607 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.4430 sec 2024-12-03T17:16:01,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on e1e0418f3d66f886e020279128ac6f55 2024-12-03T17:16:01,639 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e1e0418f3d66f886e020279128ac6f55 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T17:16:01,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=A 2024-12-03T17:16:01,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:01,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=B 2024-12-03T17:16:01,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:01,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=C 2024-12-03T17:16:01,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:01,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/f045a0c070c740e680e22bbafdd66410 is 50, key is test_row_0/A:col10/1733246161187/Put/seqid=0 2024-12-03T17:16:01,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741843_1019 (size=14341) 2024-12-03T17:16:01,728 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/f045a0c070c740e680e22bbafdd66410 2024-12-03T17:16:01,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/3093491aa9b74728abe470d574c6cd09 is 50, key is test_row_0/B:col10/1733246161187/Put/seqid=0 2024-12-03T17:16:01,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:01,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246221756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:01,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:01,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:01,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:01,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:01,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1733246221758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:01,790 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:01,790 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:01,790 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:01,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:01,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47748 deadline: 1733246221760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:01,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:01,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:01,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:01,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:01,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246221774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:01,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:01,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:01,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:01,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:01,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246221770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:01,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:01,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:01,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:01,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741844_1020 (size=12001) 2024-12-03T17:16:01,827 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/3093491aa9b74728abe470d574c6cd09 2024-12-03T17:16:01,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/682fc2860e8744f28de2f3b1d5629dbf is 50, key is test_row_0/C:col10/1733246161187/Put/seqid=0 2024-12-03T17:16:01,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741845_1021 (size=12001) 2024-12-03T17:16:01,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:01,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246221902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:01,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:01,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:01,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:01,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:01,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1733246221903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:01,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:01,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:01,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:01,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:01,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47748 deadline: 1733246221908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:01,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:01,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:01,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:01,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:01,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246221913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:01,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:01,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:01,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:01,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:01,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246221927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:01,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:01,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:01,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:02,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:02,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246222115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:02,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:02,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:02,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:02,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:02,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1733246222115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:02,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:02,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:02,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:02,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:02,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47748 deadline: 1733246222131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:02,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:02,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:02,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:02,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:02,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246222138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:02,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:02,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246222145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:02,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:02,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:02,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:02,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:02,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:02,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:02,296 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/682fc2860e8744f28de2f3b1d5629dbf 2024-12-03T17:16:02,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/f045a0c070c740e680e22bbafdd66410 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/f045a0c070c740e680e22bbafdd66410 2024-12-03T17:16:02,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/f045a0c070c740e680e22bbafdd66410, entries=200, sequenceid=62, filesize=14.0 K 2024-12-03T17:16:02,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/3093491aa9b74728abe470d574c6cd09 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/3093491aa9b74728abe470d574c6cd09 2024-12-03T17:16:02,435 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/3093491aa9b74728abe470d574c6cd09, entries=150, sequenceid=62, filesize=11.7 K 2024-12-03T17:16:02,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/682fc2860e8744f28de2f3b1d5629dbf as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/682fc2860e8744f28de2f3b1d5629dbf 2024-12-03T17:16:02,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:02,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246222434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:02,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:02,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:02,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:02,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:02,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47748 deadline: 1733246222445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:02,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:02,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:02,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:02,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:02,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:02,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1733246222444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:02,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246222464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:02,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:02,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246222464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:02,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:02,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:02,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:02,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:02,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:02,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:02,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:02,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:02,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:02,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/682fc2860e8744f28de2f3b1d5629dbf, entries=150, sequenceid=62, filesize=11.7 K 2024-12-03T17:16:02,494 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for e1e0418f3d66f886e020279128ac6f55 in 854ms, sequenceid=62, compaction requested=true 2024-12-03T17:16:02,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e1e0418f3d66f886e020279128ac6f55: 2024-12-03T17:16:02,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1e0418f3d66f886e020279128ac6f55:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T17:16:02,505 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:16:02,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:02,514 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:16:02,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1e0418f3d66f886e020279128ac6f55:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T17:16:02,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:02,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1e0418f3d66f886e020279128ac6f55:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T17:16:02,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:16:02,528 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:16:02,530 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): e1e0418f3d66f886e020279128ac6f55/B is initiating minor compaction (all files) 2024-12-03T17:16:02,530 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:16:02,531 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): e1e0418f3d66f886e020279128ac6f55/A is initiating minor compaction (all files) 2024-12-03T17:16:02,531 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1e0418f3d66f886e020279128ac6f55/B in TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:02,531 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1e0418f3d66f886e020279128ac6f55/A in TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:02,531 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/a38c5bd6befd44b8b5e0591b4b5b0c83, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/afc44c870ad044539c4a37a880dd36e1, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/3093491aa9b74728abe470d574c6cd09] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp, totalSize=35.2 K 2024-12-03T17:16:02,531 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/c65d6c13ed1942c6b9c430a51b7f0c40, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/914e83bbeaf14ece9ed23e921dbae1ca, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/f045a0c070c740e680e22bbafdd66410] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp, totalSize=37.4 K 2024-12-03T17:16:02,542 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting a38c5bd6befd44b8b5e0591b4b5b0c83, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733246159212 2024-12-03T17:16:02,542 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting c65d6c13ed1942c6b9c430a51b7f0c40, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733246159212 2024-12-03T17:16:02,546 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 914e83bbeaf14ece9ed23e921dbae1ca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733246159771 2024-12-03T17:16:02,547 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting afc44c870ad044539c4a37a880dd36e1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733246159771 2024-12-03T17:16:02,548 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 3093491aa9b74728abe470d574c6cd09, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733246161070 2024-12-03T17:16:02,548 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting f045a0c070c740e680e22bbafdd66410, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733246161070 2024-12-03T17:16:02,662 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1e0418f3d66f886e020279128ac6f55#B#compaction#9 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:16:02,664 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/f356195a9ecb4d4eb307520735db42a5 is 50, key is test_row_0/B:col10/1733246161187/Put/seqid=0 2024-12-03T17:16:02,667 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1e0418f3d66f886e020279128ac6f55#A#compaction#10 average throughput is 0.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:16:02,669 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/9eadc9e9251a4849ab6d32e94d96af73 is 50, key is test_row_0/A:col10/1733246161187/Put/seqid=0 2024-12-03T17:16:02,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741847_1023 (size=12104) 2024-12-03T17:16:02,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741846_1022 (size=12104) 2024-12-03T17:16:02,983 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e1e0418f3d66f886e020279128ac6f55 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T17:16:02,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=A 2024-12-03T17:16:02,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:02,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=B 2024-12-03T17:16:02,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:02,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=C 2024-12-03T17:16:02,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:02,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on e1e0418f3d66f886e020279128ac6f55 2024-12-03T17:16:02,996 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/1615876ac85c4b0492db1b1ea2e5e2d4 is 50, key is test_row_0/A:col10/1733246161746/Put/seqid=0 2024-12-03T17:16:03,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741848_1024 (size=14341) 2024-12-03T17:16:03,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:03,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47748 deadline: 1733246223060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:03,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:03,093 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:03,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:03,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:03,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246223060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:03,096 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/1615876ac85c4b0492db1b1ea2e5e2d4 2024-12-03T17:16:03,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:03,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:03,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:03,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:03,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1733246223092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:03,115 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:03,115 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:03,115 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:03,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:03,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246223093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:03,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:03,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246223101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:03,123 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:03,123 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:03,123 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:03,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:03,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:03,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:03,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/f321d13903b24e37a1e51de1fc4866e1 is 50, key is test_row_0/B:col10/1733246161746/Put/seqid=0 2024-12-03T17:16:03,192 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/f356195a9ecb4d4eb307520735db42a5 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/f356195a9ecb4d4eb307520735db42a5 2024-12-03T17:16:03,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:03,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246223208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:03,223 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/9eadc9e9251a4849ab6d32e94d96af73 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/9eadc9e9251a4849ab6d32e94d96af73 2024-12-03T17:16:03,225 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:03,225 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:03,225 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:03,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:03,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47748 deadline: 1733246223208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:03,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741849_1025 (size=12001) 2024-12-03T17:16:03,228 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e1e0418f3d66f886e020279128ac6f55/B of e1e0418f3d66f886e020279128ac6f55 into f356195a9ecb4d4eb307520735db42a5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:16:03,228 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1e0418f3d66f886e020279128ac6f55: 2024-12-03T17:16:03,238 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., storeName=e1e0418f3d66f886e020279128ac6f55/B, priority=13, startTime=1733246162514; duration=0sec 2024-12-03T17:16:03,238 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:16:03,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:03,239 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1e0418f3d66f886e020279128ac6f55:B 2024-12-03T17:16:03,239 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:16:03,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:03,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:03,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:03,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1733246223226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:03,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:03,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:03,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:03,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:03,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246223240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:03,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:03,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:03,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:03,257 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:16:03,257 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): e1e0418f3d66f886e020279128ac6f55/C is initiating minor compaction (all files) 2024-12-03T17:16:03,257 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1e0418f3d66f886e020279128ac6f55/C in TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:03,258 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/3077a4922e89443abbf0eca785510bf2, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/1da0d6b3fff94ce08ed729056a9c4901, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/682fc2860e8744f28de2f3b1d5629dbf] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp, totalSize=35.2 K 2024-12-03T17:16:03,259 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 3077a4922e89443abbf0eca785510bf2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733246159212 2024-12-03T17:16:03,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:03,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246223240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:03,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:03,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:03,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:03,261 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 1da0d6b3fff94ce08ed729056a9c4901, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733246159771 2024-12-03T17:16:03,264 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 682fc2860e8744f28de2f3b1d5629dbf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733246161070 2024-12-03T17:16:03,279 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e1e0418f3d66f886e020279128ac6f55/A of e1e0418f3d66f886e020279128ac6f55 into 9eadc9e9251a4849ab6d32e94d96af73(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:16:03,279 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1e0418f3d66f886e020279128ac6f55: 2024-12-03T17:16:03,279 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., storeName=e1e0418f3d66f886e020279128ac6f55/A, priority=13, startTime=1733246162498; duration=0sec 2024-12-03T17:16:03,279 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:03,279 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1e0418f3d66f886e020279128ac6f55:A 2024-12-03T17:16:03,300 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1e0418f3d66f886e020279128ac6f55#C#compaction#13 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:16:03,304 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/52dc01b1874e4939b1bab5303281d6a6 is 50, key is test_row_0/C:col10/1733246161187/Put/seqid=0 2024-12-03T17:16:03,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T17:16:03,345 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:16:03,351 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T17:16:03,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-12-03T17:16:03,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741850_1026 (size=12104) 2024-12-03T17:16:03,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-03T17:16:03,375 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T17:16:03,383 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T17:16:03,383 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T17:16:03,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:03,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246223434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:03,449 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:03,449 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:03,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:03,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:03,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47748 deadline: 1733246223444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:03,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:03,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:03,464 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:03,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-03T17:16:03,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:03,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1733246223464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:03,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:03,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:03,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:03,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:03,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246223466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:03,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:03,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:03,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:03,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:03,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246223474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:03,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:03,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:03,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:03,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-03T17:16:03,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:03,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. as already flushing 2024-12-03T17:16:03,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:03,560 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:03,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:03,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:03,629 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/f321d13903b24e37a1e51de1fc4866e1 2024-12-03T17:16:03,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/4327c46957304386915c494d6aa01609 is 50, key is test_row_0/C:col10/1733246161746/Put/seqid=0 2024-12-03T17:16:03,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-03T17:16:03,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741851_1027 (size=12001) 2024-12-03T17:16:03,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-03T17:16:03,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:03,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. as already flushing 2024-12-03T17:16:03,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:03,717 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:03,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:03,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:03,723 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/4327c46957304386915c494d6aa01609 2024-12-03T17:16:03,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:03,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246223755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:03,763 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:03,763 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:03,763 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:03,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/1615876ac85c4b0492db1b1ea2e5e2d4 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/1615876ac85c4b0492db1b1ea2e5e2d4 2024-12-03T17:16:03,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:03,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47748 deadline: 1733246223775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:03,792 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:03,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:03,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:03,797 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/1615876ac85c4b0492db1b1ea2e5e2d4, entries=200, sequenceid=82, filesize=14.0 K 2024-12-03T17:16:03,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:03,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:03,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1733246223785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:03,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246223795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:03,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:03,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:03,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:03,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:03,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246223804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:03,812 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/52dc01b1874e4939b1bab5303281d6a6 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/52dc01b1874e4939b1bab5303281d6a6 2024-12-03T17:16:03,819 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:03,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/f321d13903b24e37a1e51de1fc4866e1 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/f321d13903b24e37a1e51de1fc4866e1 2024-12-03T17:16:03,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:03,819 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:03,819 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:03,819 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:03,819 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:03,852 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e1e0418f3d66f886e020279128ac6f55/C of e1e0418f3d66f886e020279128ac6f55 into 52dc01b1874e4939b1bab5303281d6a6(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:16:03,853 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1e0418f3d66f886e020279128ac6f55: 2024-12-03T17:16:03,853 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., storeName=e1e0418f3d66f886e020279128ac6f55/C, priority=13, startTime=1733246162516; duration=0sec 2024-12-03T17:16:03,853 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:03,853 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1e0418f3d66f886e020279128ac6f55:C 2024-12-03T17:16:03,880 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-03T17:16:03,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:03,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. as already flushing 2024-12-03T17:16:03,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:03,881 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:03,881 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/f321d13903b24e37a1e51de1fc4866e1, entries=150, sequenceid=82, filesize=11.7 K 2024-12-03T17:16:03,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:03,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:03,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/4327c46957304386915c494d6aa01609 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/4327c46957304386915c494d6aa01609 2024-12-03T17:16:03,962 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/4327c46957304386915c494d6aa01609, entries=150, sequenceid=82, filesize=11.7 K 2024-12-03T17:16:03,971 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for e1e0418f3d66f886e020279128ac6f55 in 988ms, sequenceid=82, compaction requested=false 2024-12-03T17:16:03,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e1e0418f3d66f886e020279128ac6f55: 2024-12-03T17:16:03,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-03T17:16:04,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-03T17:16:04,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:04,048 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing e1e0418f3d66f886e020279128ac6f55 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T17:16:04,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=A 2024-12-03T17:16:04,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:04,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=B 2024-12-03T17:16:04,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:04,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=C 2024-12-03T17:16:04,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:04,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/9f3f9df63327452da33b9bcd020d6a29 is 50, key is test_row_0/A:col10/1733246163058/Put/seqid=0 2024-12-03T17:16:04,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741852_1028 (size=12001) 2024-12-03T17:16:04,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on e1e0418f3d66f886e020279128ac6f55 2024-12-03T17:16:04,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. as already flushing 2024-12-03T17:16:04,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:04,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246224379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:04,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:04,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:04,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:04,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:04,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246224382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:04,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:04,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47748 deadline: 1733246224387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:04,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:04,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:04,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:04,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:04,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:04,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:04,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:04,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1733246224397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:04,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:04,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246224400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:04,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:04,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:04,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:04,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:04,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:04,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:04,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-03T17:16:04,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:04,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:04,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246224505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:04,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47748 deadline: 1733246224505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:04,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:04,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:04,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:04,508 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:04,508 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:04,508 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:04,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:04,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246224515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:04,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:04,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:04,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:04,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:04,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246224524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:04,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:04,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:04,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:04,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:04,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1733246224524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:04,531 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:04,531 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:04,531 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:04,565 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/9f3f9df63327452da33b9bcd020d6a29 2024-12-03T17:16:04,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/d773d21cc5ee4e9d9db9641dd7c2bbb4 is 50, key is test_row_0/B:col10/1733246163058/Put/seqid=0 2024-12-03T17:16:04,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741853_1029 (size=12001) 2024-12-03T17:16:04,678 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/d773d21cc5ee4e9d9db9641dd7c2bbb4 2024-12-03T17:16:04,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:04,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47748 deadline: 1733246224715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:04,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:04,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:04,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:04,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:04,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246224715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:04,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:04,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:04,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:04,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/402a04bfd59247d396c26265a5744bca is 50, key is test_row_0/C:col10/1733246163058/Put/seqid=0 2024-12-03T17:16:04,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:04,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246224735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:04,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:04,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:04,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:04,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:04,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246224735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:04,739 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:04,739 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:04,739 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:04,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:04,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1733246224735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:04,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:04,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:04,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:04,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741854_1030 (size=12001) 2024-12-03T17:16:05,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:05,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47748 deadline: 1733246225035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:05,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:05,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:05,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:05,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:05,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246225045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:05,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:05,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:05,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:05,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:05,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246225045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:05,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:05,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:05,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:05,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:05,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246225046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:05,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:05,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:05,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:05,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:05,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1733246225055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:05,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:05,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:05,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:05,203 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/402a04bfd59247d396c26265a5744bca 2024-12-03T17:16:05,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/9f3f9df63327452da33b9bcd020d6a29 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/9f3f9df63327452da33b9bcd020d6a29 2024-12-03T17:16:05,270 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/9f3f9df63327452da33b9bcd020d6a29, entries=150, sequenceid=98, filesize=11.7 K 2024-12-03T17:16:05,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/d773d21cc5ee4e9d9db9641dd7c2bbb4 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/d773d21cc5ee4e9d9db9641dd7c2bbb4 2024-12-03T17:16:05,297 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/d773d21cc5ee4e9d9db9641dd7c2bbb4, entries=150, sequenceid=98, filesize=11.7 K 2024-12-03T17:16:05,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/402a04bfd59247d396c26265a5744bca as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/402a04bfd59247d396c26265a5744bca 2024-12-03T17:16:05,350 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/402a04bfd59247d396c26265a5744bca, entries=150, sequenceid=98, filesize=11.7 K 2024-12-03T17:16:05,352 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for e1e0418f3d66f886e020279128ac6f55 in 1303ms, sequenceid=98, compaction requested=true 2024-12-03T17:16:05,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for e1e0418f3d66f886e020279128ac6f55: 2024-12-03T17:16:05,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:05,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-03T17:16:05,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-12-03T17:16:05,365 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-03T17:16:05,365 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9740 sec 2024-12-03T17:16:05,374 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 2.0150 sec 2024-12-03T17:16:05,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-03T17:16:05,515 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:16:05,519 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T17:16:05,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-12-03T17:16:05,525 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T17:16:05,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-03T17:16:05,528 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T17:16:05,528 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T17:16:05,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on e1e0418f3d66f886e020279128ac6f55 2024-12-03T17:16:05,587 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e1e0418f3d66f886e020279128ac6f55 3/3 column families, dataSize=127.47 KB heapSize=379.97 KB 2024-12-03T17:16:05,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=A 2024-12-03T17:16:05,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:05,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=B 2024-12-03T17:16:05,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:05,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=C 2024-12-03T17:16:05,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:05,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/f4e6db0ca4604b388836071c0cce301e is 50, key is test_row_0/A:col10/1733246165559/Put/seqid=0 2024-12-03T17:16:05,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:05,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246225624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:05,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:05,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:05,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:05,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246225625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:05,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:05,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-03T17:16:05,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:05,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:05,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:05,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:05,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47748 deadline: 1733246225626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:05,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:05,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246225632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:05,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:05,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:05,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:05,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:05,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1733246225638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:05,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:05,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:05,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:05,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:05,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:05,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:05,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741855_1031 (size=12001) 2024-12-03T17:16:05,664 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/f4e6db0ca4604b388836071c0cce301e 2024-12-03T17:16:05,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-03T17:16:05,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:05,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. as already flushing 2024-12-03T17:16:05,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:05,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:05,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:05,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:05,727 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/b78e406e719649338ba157358c80f6e7 is 50, key is test_row_0/B:col10/1733246165559/Put/seqid=0 2024-12-03T17:16:05,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:05,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:05,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246225744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:05,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246225744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:05,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:05,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47748 deadline: 1733246225749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:05,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:05,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:05,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:05,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:05,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:05,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:05,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:05,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:05,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:05,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:05,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1733246225757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:05,761 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:05,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:05,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:05,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:05,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246225759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:05,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:05,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:05,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:05,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741856_1032 (size=12001) 2024-12-03T17:16:05,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-03T17:16:05,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:05,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-03T17:16:05,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. as already flushing 2024-12-03T17:16:05,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:05,844 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:05,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:05,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:05,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:05,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47748 deadline: 1733246225964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:05,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:05,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:05,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:05,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:05,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246225964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:05,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:05,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246225965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:05,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:05,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:05,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:05,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:05,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1733246225966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:05,972 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:05,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:05,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:05,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:05,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:05,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:05,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:05,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246225979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:05,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:05,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:05,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:06,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-03T17:16:06,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:06,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. as already flushing 2024-12-03T17:16:06,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:06,007 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:06,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:06,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:06,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-03T17:16:06,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-03T17:16:06,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:06,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. as already flushing 2024-12-03T17:16:06,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:06,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:06,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:06,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:06,184 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/b78e406e719649338ba157358c80f6e7 2024-12-03T17:16:06,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/065813df88d6491bb396a00c79d9aa44 is 50, key is test_row_0/C:col10/1733246165559/Put/seqid=0 2024-12-03T17:16:06,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:06,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47748 deadline: 1733246226278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:06,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:06,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246226285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:06,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:06,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:06,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:06,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:06,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:06,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:06,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:06,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246226285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:06,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:06,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1733246226292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:06,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:06,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:06,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:06,302 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:06,302 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:06,303 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:06,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741857_1033 (size=12001) 2024-12-03T17:16:06,315 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/065813df88d6491bb396a00c79d9aa44 2024-12-03T17:16:06,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:06,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246226304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:06,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:06,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:06,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:06,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-03T17:16:06,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:06,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. as already flushing 2024-12-03T17:16:06,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:06,340 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:06,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:06,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/f4e6db0ca4604b388836071c0cce301e as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/f4e6db0ca4604b388836071c0cce301e 2024-12-03T17:16:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:06,374 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/f4e6db0ca4604b388836071c0cce301e, entries=150, sequenceid=121, filesize=11.7 K 2024-12-03T17:16:06,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/b78e406e719649338ba157358c80f6e7 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/b78e406e719649338ba157358c80f6e7 2024-12-03T17:16:06,399 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/b78e406e719649338ba157358c80f6e7, entries=150, sequenceid=121, filesize=11.7 K 2024-12-03T17:16:06,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/065813df88d6491bb396a00c79d9aa44 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/065813df88d6491bb396a00c79d9aa44 2024-12-03T17:16:06,420 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/065813df88d6491bb396a00c79d9aa44, entries=150, sequenceid=121, filesize=11.7 K 2024-12-03T17:16:06,423 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~399.84 KB/409440, currentSize=46.96 KB/48090 for e1e0418f3d66f886e020279128ac6f55 in 836ms, sequenceid=121, compaction requested=true 2024-12-03T17:16:06,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e1e0418f3d66f886e020279128ac6f55: 2024-12-03T17:16:06,425 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T17:16:06,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1e0418f3d66f886e020279128ac6f55:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T17:16:06,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:06,426 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T17:16:06,436 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50447 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T17:16:06,436 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): e1e0418f3d66f886e020279128ac6f55/A is initiating minor compaction (all files) 2024-12-03T17:16:06,436 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1e0418f3d66f886e020279128ac6f55/A in TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:06,436 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T17:16:06,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1e0418f3d66f886e020279128ac6f55:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T17:16:06,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:06,437 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): e1e0418f3d66f886e020279128ac6f55/B is initiating minor compaction (all files) 2024-12-03T17:16:06,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1e0418f3d66f886e020279128ac6f55:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T17:16:06,437 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/9eadc9e9251a4849ab6d32e94d96af73, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/1615876ac85c4b0492db1b1ea2e5e2d4, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/9f3f9df63327452da33b9bcd020d6a29, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/f4e6db0ca4604b388836071c0cce301e] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp, totalSize=49.3 K 2024-12-03T17:16:06,437 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1e0418f3d66f886e020279128ac6f55/B in TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:06,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:16:06,437 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/f356195a9ecb4d4eb307520735db42a5, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/f321d13903b24e37a1e51de1fc4866e1, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/d773d21cc5ee4e9d9db9641dd7c2bbb4, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/b78e406e719649338ba157358c80f6e7] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp, totalSize=47.0 K 2024-12-03T17:16:06,438 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 9eadc9e9251a4849ab6d32e94d96af73, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733246161070 2024-12-03T17:16:06,438 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting f356195a9ecb4d4eb307520735db42a5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733246161070 2024-12-03T17:16:06,439 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 1615876ac85c4b0492db1b1ea2e5e2d4, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733246161746 2024-12-03T17:16:06,440 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting f321d13903b24e37a1e51de1fc4866e1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733246161746 2024-12-03T17:16:06,440 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f3f9df63327452da33b9bcd020d6a29, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733246163058 2024-12-03T17:16:06,440 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting d773d21cc5ee4e9d9db9641dd7c2bbb4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733246163058 2024-12-03T17:16:06,441 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting f4e6db0ca4604b388836071c0cce301e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1733246165559 2024-12-03T17:16:06,441 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting b78e406e719649338ba157358c80f6e7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1733246165559 2024-12-03T17:16:06,480 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1e0418f3d66f886e020279128ac6f55#B#compaction#21 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:16:06,481 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/e683ac9734f74f5f98a549bb0961b0b2 is 50, key is test_row_0/B:col10/1733246165559/Put/seqid=0 2024-12-03T17:16:06,491 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1e0418f3d66f886e020279128ac6f55#A#compaction#22 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:16:06,492 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/cdbf3bd3ce2c432bbbe1722d094f228b is 50, key is test_row_0/A:col10/1733246165559/Put/seqid=0 2024-12-03T17:16:06,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-03T17:16:06,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:06,500 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing e1e0418f3d66f886e020279128ac6f55 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T17:16:06,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=A 2024-12-03T17:16:06,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:06,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=B 2024-12-03T17:16:06,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:06,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=C 2024-12-03T17:16:06,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:06,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741859_1035 (size=12241) 2024-12-03T17:16:06,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741858_1034 (size=12241) 2024-12-03T17:16:06,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/0758a8578dda425f94814140139181d3 is 50, key is test_row_0/A:col10/1733246165623/Put/seqid=0 2024-12-03T17:16:06,538 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/e683ac9734f74f5f98a549bb0961b0b2 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/e683ac9734f74f5f98a549bb0961b0b2 2024-12-03T17:16:06,554 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e1e0418f3d66f886e020279128ac6f55/B of e1e0418f3d66f886e020279128ac6f55 into e683ac9734f74f5f98a549bb0961b0b2(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:16:06,554 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1e0418f3d66f886e020279128ac6f55: 2024-12-03T17:16:06,554 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., storeName=e1e0418f3d66f886e020279128ac6f55/B, priority=12, startTime=1733246166426; duration=0sec 2024-12-03T17:16:06,555 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:16:06,555 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1e0418f3d66f886e020279128ac6f55:B 2024-12-03T17:16:06,555 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T17:16:06,559 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T17:16:06,559 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): e1e0418f3d66f886e020279128ac6f55/C is initiating minor compaction (all files) 2024-12-03T17:16:06,560 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1e0418f3d66f886e020279128ac6f55/C in TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:06,560 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/52dc01b1874e4939b1bab5303281d6a6, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/4327c46957304386915c494d6aa01609, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/402a04bfd59247d396c26265a5744bca, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/065813df88d6491bb396a00c79d9aa44] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp, totalSize=47.0 K 2024-12-03T17:16:06,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741860_1036 (size=12101) 2024-12-03T17:16:06,566 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52dc01b1874e4939b1bab5303281d6a6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733246161070 2024-12-03T17:16:06,566 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/0758a8578dda425f94814140139181d3 2024-12-03T17:16:06,567 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4327c46957304386915c494d6aa01609, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733246161746 2024-12-03T17:16:06,571 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 402a04bfd59247d396c26265a5744bca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733246163058 2024-12-03T17:16:06,576 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 065813df88d6491bb396a00c79d9aa44, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1733246165559 2024-12-03T17:16:06,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/402986351b3c4b02a960ea59c0006f53 is 50, key is test_row_0/B:col10/1733246165623/Put/seqid=0 2024-12-03T17:16:06,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-03T17:16:06,670 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1e0418f3d66f886e020279128ac6f55#C#compaction#25 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:16:06,671 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/76d9ffdc5eaa41d1b6e921ba50ccb617 is 50, key is test_row_0/C:col10/1733246165559/Put/seqid=0 2024-12-03T17:16:06,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741861_1037 (size=12101) 2024-12-03T17:16:06,688 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/402986351b3c4b02a960ea59c0006f53 2024-12-03T17:16:06,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741862_1038 (size=12241) 2024-12-03T17:16:06,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/1d1f4a0bae6d4630a88665261a971701 is 50, key is test_row_0/C:col10/1733246165623/Put/seqid=0 2024-12-03T17:16:06,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741863_1039 (size=12101) 2024-12-03T17:16:06,801 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/1d1f4a0bae6d4630a88665261a971701 2024-12-03T17:16:06,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on e1e0418f3d66f886e020279128ac6f55 2024-12-03T17:16:06,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. as already flushing 2024-12-03T17:16:06,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/0758a8578dda425f94814140139181d3 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/0758a8578dda425f94814140139181d3 2024-12-03T17:16:06,891 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/0758a8578dda425f94814140139181d3, entries=150, sequenceid=131, filesize=11.8 K 2024-12-03T17:16:06,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/402986351b3c4b02a960ea59c0006f53 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/402986351b3c4b02a960ea59c0006f53 2024-12-03T17:16:06,913 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/402986351b3c4b02a960ea59c0006f53, entries=150, sequenceid=131, filesize=11.8 K 2024-12-03T17:16:06,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/1d1f4a0bae6d4630a88665261a971701 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/1d1f4a0bae6d4630a88665261a971701 2024-12-03T17:16:06,938 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/1d1f4a0bae6d4630a88665261a971701, entries=150, sequenceid=131, filesize=11.8 K 2024-12-03T17:16:06,940 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=114.05 KB/116790 for e1e0418f3d66f886e020279128ac6f55 in 440ms, sequenceid=131, compaction requested=false 2024-12-03T17:16:06,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for e1e0418f3d66f886e020279128ac6f55: 2024-12-03T17:16:06,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:06,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-03T17:16:06,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-12-03T17:16:06,948 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-03T17:16:06,948 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4160 sec 2024-12-03T17:16:06,951 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.4300 sec 2024-12-03T17:16:06,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on e1e0418f3d66f886e020279128ac6f55 2024-12-03T17:16:06,956 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e1e0418f3d66f886e020279128ac6f55 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-03T17:16:06,956 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/cdbf3bd3ce2c432bbbe1722d094f228b as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/cdbf3bd3ce2c432bbbe1722d094f228b 2024-12-03T17:16:06,957 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733246166914, max timestamp=1733246166926Num uniques 859; 2024-12-03T17:16:06,957 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T17:16:06,961 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733246166914, max timestamp=1733246166938Num uniques 859; 2024-12-03T17:16:06,962 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-03T17:16:06,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=A 2024-12-03T17:16:06,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:06,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=B 2024-12-03T17:16:06,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:06,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=C 2024-12-03T17:16:06,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:06,977 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/4c94eb44dfef4b7fac4092c4fc03f77a is 50, key is test_row_0/A:col10/1733246166918/Put/seqid=0 2024-12-03T17:16:06,984 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e1e0418f3d66f886e020279128ac6f55/A of e1e0418f3d66f886e020279128ac6f55 into cdbf3bd3ce2c432bbbe1722d094f228b(size=12.0 K), total size for store is 23.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:16:06,984 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1e0418f3d66f886e020279128ac6f55: 2024-12-03T17:16:06,984 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., storeName=e1e0418f3d66f886e020279128ac6f55/A, priority=12, startTime=1733246166423; duration=0sec 2024-12-03T17:16:06,985 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:06,985 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1e0418f3d66f886e020279128ac6f55:A 2024-12-03T17:16:07,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741864_1040 (size=14541) 2024-12-03T17:16:07,019 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/4c94eb44dfef4b7fac4092c4fc03f77a 2024-12-03T17:16:07,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/696fff161e5f41a58746827b44d9d9a5 is 50, key is test_row_0/B:col10/1733246166918/Put/seqid=0 2024-12-03T17:16:07,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741865_1041 (size=12151) 2024-12-03T17:16:07,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:07,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246227044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:07,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:07,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:07,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:07,070 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.02 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/696fff161e5f41a58746827b44d9d9a5 2024-12-03T17:16:07,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:07,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246227057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:07,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:07,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:07,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:07,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:07,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246227064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:07,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:07,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:07,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:07,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:07,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1733246227069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:07,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:07,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47748 deadline: 1733246227071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:07,095 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:07,095 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:07,095 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:07,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:07,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:07,096 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:07,115 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/d60c16f0d1ee49479791200b763e456a is 50, key is test_row_0/C:col10/1733246166918/Put/seqid=0 2024-12-03T17:16:07,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741866_1042 (size=12151) 2024-12-03T17:16:07,154 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/76d9ffdc5eaa41d1b6e921ba50ccb617 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/76d9ffdc5eaa41d1b6e921ba50ccb617 2024-12-03T17:16:07,155 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/d60c16f0d1ee49479791200b763e456a 2024-12-03T17:16:07,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/4c94eb44dfef4b7fac4092c4fc03f77a as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/4c94eb44dfef4b7fac4092c4fc03f77a 2024-12-03T17:16:07,179 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e1e0418f3d66f886e020279128ac6f55/C of e1e0418f3d66f886e020279128ac6f55 into 76d9ffdc5eaa41d1b6e921ba50ccb617(size=12.0 K), total size for store is 23.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:16:07,179 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1e0418f3d66f886e020279128ac6f55: 2024-12-03T17:16:07,180 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., storeName=e1e0418f3d66f886e020279128ac6f55/C, priority=12, startTime=1733246166437; duration=0sec 2024-12-03T17:16:07,180 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:07,180 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1e0418f3d66f886e020279128ac6f55:C 2024-12-03T17:16:07,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/4c94eb44dfef4b7fac4092c4fc03f77a, entries=200, sequenceid=155, filesize=14.2 K 2024-12-03T17:16:07,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:07,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246227178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:07,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/696fff161e5f41a58746827b44d9d9a5 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/696fff161e5f41a58746827b44d9d9a5 2024-12-03T17:16:07,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:07,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246227178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:07,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:07,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:07,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:07,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:07,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:07,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:07,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/696fff161e5f41a58746827b44d9d9a5, entries=150, sequenceid=155, filesize=11.9 K 2024-12-03T17:16:07,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/d60c16f0d1ee49479791200b763e456a as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/d60c16f0d1ee49479791200b763e456a 2024-12-03T17:16:07,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:07,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246227193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:07,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:07,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:07,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:07,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:07,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1733246227204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:07,219 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:07,219 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:07,219 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:07,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:07,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47748 deadline: 1733246227210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:07,225 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/d60c16f0d1ee49479791200b763e456a, entries=150, sequenceid=155, filesize=11.9 K 2024-12-03T17:16:07,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:07,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:07,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:07,233 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~71.17 KB/72875, heapSize ~203.05 KB/207928, currentSize=107.34 KB/109920 for e1e0418f3d66f886e020279128ac6f55 in 277ms, sequenceid=155, compaction requested=true 2024-12-03T17:16:07,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e1e0418f3d66f886e020279128ac6f55: 2024-12-03T17:16:07,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1e0418f3d66f886e020279128ac6f55:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T17:16:07,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:07,233 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:16:07,233 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:16:07,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1e0418f3d66f886e020279128ac6f55:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T17:16:07,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:07,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1e0418f3d66f886e020279128ac6f55:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T17:16:07,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:16:07,237 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36493 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:16:07,237 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): e1e0418f3d66f886e020279128ac6f55/B is initiating minor compaction (all files) 2024-12-03T17:16:07,237 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1e0418f3d66f886e020279128ac6f55/B in TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:07,237 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/e683ac9734f74f5f98a549bb0961b0b2, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/402986351b3c4b02a960ea59c0006f53, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/696fff161e5f41a58746827b44d9d9a5] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp, totalSize=35.6 K 2024-12-03T17:16:07,238 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38883 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:16:07,238 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): e1e0418f3d66f886e020279128ac6f55/A is initiating minor compaction (all files) 2024-12-03T17:16:07,238 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1e0418f3d66f886e020279128ac6f55/A in TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:07,238 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/cdbf3bd3ce2c432bbbe1722d094f228b, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/0758a8578dda425f94814140139181d3, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/4c94eb44dfef4b7fac4092c4fc03f77a] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp, totalSize=38.0 K 2024-12-03T17:16:07,239 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting e683ac9734f74f5f98a549bb0961b0b2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1733246165559 2024-12-03T17:16:07,240 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 402986351b3c4b02a960ea59c0006f53, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733246165620 2024-12-03T17:16:07,241 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting cdbf3bd3ce2c432bbbe1722d094f228b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1733246165559 2024-12-03T17:16:07,244 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 696fff161e5f41a58746827b44d9d9a5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733246166918 2024-12-03T17:16:07,245 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0758a8578dda425f94814140139181d3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733246165620 2024-12-03T17:16:07,246 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c94eb44dfef4b7fac4092c4fc03f77a, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733246166918 2024-12-03T17:16:07,265 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1e0418f3d66f886e020279128ac6f55#B#compaction#30 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:16:07,267 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/71278d223dc8459dacc71bb962c71dda is 50, key is test_row_0/B:col10/1733246166918/Put/seqid=0 2024-12-03T17:16:07,280 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1e0418f3d66f886e020279128ac6f55#A#compaction#31 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:16:07,282 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/7fd11e6dc0d145e09dcc71387c2afa6e is 50, key is test_row_0/A:col10/1733246166918/Put/seqid=0 2024-12-03T17:16:07,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741868_1044 (size=12493) 2024-12-03T17:16:07,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741867_1043 (size=12493) 2024-12-03T17:16:07,361 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/7fd11e6dc0d145e09dcc71387c2afa6e as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/7fd11e6dc0d145e09dcc71387c2afa6e 2024-12-03T17:16:07,379 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/71278d223dc8459dacc71bb962c71dda as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/71278d223dc8459dacc71bb962c71dda 2024-12-03T17:16:07,398 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e1e0418f3d66f886e020279128ac6f55/A of e1e0418f3d66f886e020279128ac6f55 into 7fd11e6dc0d145e09dcc71387c2afa6e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:16:07,398 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1e0418f3d66f886e020279128ac6f55: 2024-12-03T17:16:07,398 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., storeName=e1e0418f3d66f886e020279128ac6f55/A, priority=13, startTime=1733246167233; duration=0sec 2024-12-03T17:16:07,399 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:16:07,399 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1e0418f3d66f886e020279128ac6f55:A 2024-12-03T17:16:07,399 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:16:07,406 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36493 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:16:07,406 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): e1e0418f3d66f886e020279128ac6f55/C is initiating minor compaction (all files) 2024-12-03T17:16:07,406 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1e0418f3d66f886e020279128ac6f55/C in TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:07,406 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/76d9ffdc5eaa41d1b6e921ba50ccb617, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/1d1f4a0bae6d4630a88665261a971701, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/d60c16f0d1ee49479791200b763e456a] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp, totalSize=35.6 K 2024-12-03T17:16:07,407 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76d9ffdc5eaa41d1b6e921ba50ccb617, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1733246165559 2024-12-03T17:16:07,411 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d1f4a0bae6d4630a88665261a971701, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733246165620 2024-12-03T17:16:07,412 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting d60c16f0d1ee49479791200b763e456a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733246166918 2024-12-03T17:16:07,415 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e1e0418f3d66f886e020279128ac6f55 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T17:16:07,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on e1e0418f3d66f886e020279128ac6f55 2024-12-03T17:16:07,420 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e1e0418f3d66f886e020279128ac6f55/B of e1e0418f3d66f886e020279128ac6f55 into 71278d223dc8459dacc71bb962c71dda(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:16:07,420 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1e0418f3d66f886e020279128ac6f55: 2024-12-03T17:16:07,420 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., storeName=e1e0418f3d66f886e020279128ac6f55/B, priority=13, startTime=1733246167233; duration=0sec 2024-12-03T17:16:07,420 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:07,420 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1e0418f3d66f886e020279128ac6f55:B 2024-12-03T17:16:07,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=A 2024-12-03T17:16:07,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:07,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=B 2024-12-03T17:16:07,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:07,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=C 2024-12-03T17:16:07,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:07,438 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1e0418f3d66f886e020279128ac6f55#C#compaction#32 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:16:07,439 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/5e85c2c97bec403ca33238f586df759a is 50, key is test_row_0/C:col10/1733246166918/Put/seqid=0 2024-12-03T17:16:07,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/44db2c1cf0f046fbb654d96b2e6c2ea3 is 50, key is test_row_0/A:col10/1733246167408/Put/seqid=0 2024-12-03T17:16:07,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:07,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246227458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:07,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:07,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:07,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:07,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:07,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246227458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:07,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:07,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:07,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:07,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:07,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1733246227464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:07,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:07,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246227467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:07,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:07,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:07,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:07,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:07,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47748 deadline: 1733246227472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:07,482 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:07,482 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:07,482 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:07,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:07,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:07,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:07,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741869_1045 (size=12493) 2024-12-03T17:16:07,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741870_1046 (size=14541) 2024-12-03T17:16:07,528 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/44db2c1cf0f046fbb654d96b2e6c2ea3 2024-12-03T17:16:07,546 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/5e85c2c97bec403ca33238f586df759a as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/5e85c2c97bec403ca33238f586df759a 2024-12-03T17:16:07,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/cb3f2576f3ad4faa8571ef8d07d51a82 is 50, key is test_row_0/B:col10/1733246167408/Put/seqid=0 2024-12-03T17:16:07,570 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e1e0418f3d66f886e020279128ac6f55/C of e1e0418f3d66f886e020279128ac6f55 into 5e85c2c97bec403ca33238f586df759a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:16:07,570 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1e0418f3d66f886e020279128ac6f55: 2024-12-03T17:16:07,570 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., storeName=e1e0418f3d66f886e020279128ac6f55/C, priority=13, startTime=1733246167234; duration=0sec 2024-12-03T17:16:07,570 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:07,570 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1e0418f3d66f886e020279128ac6f55:C 2024-12-03T17:16:07,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:07,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246227574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:07,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:07,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246227584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:07,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:07,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:07,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:07,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:07,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:07,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:07,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:07,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246227585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:07,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:07,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:07,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:07,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741871_1047 (size=12151) 2024-12-03T17:16:07,597 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/cb3f2576f3ad4faa8571ef8d07d51a82 2024-12-03T17:16:07,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:07,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47748 deadline: 1733246227594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:07,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:07,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:07,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:07,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:07,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1733246227596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:07,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:07,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:07,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:07,625 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/d49cc80422434fbaba448bdd5a1db9ce is 50, key is test_row_0/C:col10/1733246167408/Put/seqid=0 2024-12-03T17:16:07,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-03T17:16:07,689 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:16:07,690 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T17:16:07,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-12-03T17:16:07,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T17:16:07,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741872_1048 (size=12151) 2024-12-03T17:16:07,704 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T17:16:07,705 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T17:16:07,706 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T17:16:07,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T17:16:07,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:07,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246227798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:07,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:07,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:07,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:07,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:07,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246227804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:07,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:07,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:07,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:07,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:07,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246227804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:07,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:07,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:07,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:07,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:07,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47748 deadline: 1733246227813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:07,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:07,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1733246227814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:07,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:07,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:07,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:07,820 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:07,820 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:07,820 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:07,867 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-03T17:16:07,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:07,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. as already flushing 2024-12-03T17:16:07,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:07,867 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:07,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:07,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:08,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T17:16:08,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-03T17:16:08,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:08,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. as already flushing 2024-12-03T17:16:08,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:08,033 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:08,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:08,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:08,107 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/d49cc80422434fbaba448bdd5a1db9ce 2024-12-03T17:16:08,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:08,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246228115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:08,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:08,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:08,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:08,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/44db2c1cf0f046fbb654d96b2e6c2ea3 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/44db2c1cf0f046fbb654d96b2e6c2ea3 2024-12-03T17:16:08,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/44db2c1cf0f046fbb654d96b2e6c2ea3, entries=200, sequenceid=180, filesize=14.2 K 2024-12-03T17:16:08,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/cb3f2576f3ad4faa8571ef8d07d51a82 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/cb3f2576f3ad4faa8571ef8d07d51a82 2024-12-03T17:16:08,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:08,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246228127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:08,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:08,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47748 deadline: 1733246228128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:08,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:08,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:08,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:08,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:08,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:08,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:08,146 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/cb3f2576f3ad4faa8571ef8d07d51a82, entries=150, sequenceid=180, filesize=11.9 K 2024-12-03T17:16:08,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/d49cc80422434fbaba448bdd5a1db9ce as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/d49cc80422434fbaba448bdd5a1db9ce 2024-12-03T17:16:08,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:08,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246228134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:08,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:08,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:08,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:08,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:08,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1733246228135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:08,158 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:08,158 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:08,158 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:08,173 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/d49cc80422434fbaba448bdd5a1db9ce, entries=150, sequenceid=180, filesize=11.9 K 2024-12-03T17:16:08,183 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for e1e0418f3d66f886e020279128ac6f55 in 765ms, sequenceid=180, compaction requested=false 2024-12-03T17:16:08,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e1e0418f3d66f886e020279128ac6f55: 2024-12-03T17:16:08,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-03T17:16:08,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:08,223 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing e1e0418f3d66f886e020279128ac6f55 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-03T17:16:08,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=A 2024-12-03T17:16:08,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:08,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=B 2024-12-03T17:16:08,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:08,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=C 2024-12-03T17:16:08,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:08,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/b9efbacfc85549de96aad588f52eb9a6 is 50, key is test_row_0/A:col10/1733246167464/Put/seqid=0 2024-12-03T17:16:08,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741873_1049 (size=12151) 2024-12-03T17:16:08,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T17:16:08,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. as already flushing 2024-12-03T17:16:08,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on e1e0418f3d66f886e020279128ac6f55 2024-12-03T17:16:08,739 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/b9efbacfc85549de96aad588f52eb9a6 2024-12-03T17:16:08,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733246168664, max timestamp=1733246168748Num uniques 859; 2024-12-03T17:16:08,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733246168664, max timestamp=1733246168748Num uniques 859; 2024-12-03T17:16:08,754 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T17:16:08,754 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T17:16:08,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/c38d66640f0643b0b51ce5dcc7adc7ec is 50, key is test_row_0/B:col10/1733246167464/Put/seqid=0 2024-12-03T17:16:08,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:08,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47748 deadline: 1733246228778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:08,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:08,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1733246228782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:08,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:08,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:08,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:08,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:08,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:08,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:08,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:08,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246228799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:08,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:08,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:08,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:08,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:08,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246228802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:08,812 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:08,812 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:08,812 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:08,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:08,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246228808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:08,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:08,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:08,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:08,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741874_1050 (size=12151) 2024-12-03T17:16:08,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T17:16:08,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:08,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246228916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:08,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:08,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:08,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:08,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:08,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246228916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:08,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:08,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47748 deadline: 1733246228916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:08,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:08,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1733246228920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:08,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:08,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:08,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:08,923 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:08,923 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:08,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:08,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:08,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:08,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:08,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:08,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246228924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:08,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:08,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:08,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:09,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:09,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1733246229134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:09,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:09,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:09,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:09,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246229134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:09,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:09,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:09,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246229135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:09,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:09,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:09,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:09,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:09,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:09,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:09,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:09,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47748 deadline: 1733246229135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:09,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:09,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:09,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:09,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:09,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246229155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:09,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:09,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:09,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:09,266 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/c38d66640f0643b0b51ce5dcc7adc7ec 2024-12-03T17:16:09,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/368af452cbc2404f9eff521d572442ac is 50, key is test_row_0/C:col10/1733246167464/Put/seqid=0 2024-12-03T17:16:09,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741875_1051 (size=12151) 2024-12-03T17:16:09,352 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/368af452cbc2404f9eff521d572442ac 2024-12-03T17:16:09,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/b9efbacfc85549de96aad588f52eb9a6 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/b9efbacfc85549de96aad588f52eb9a6 2024-12-03T17:16:09,379 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/b9efbacfc85549de96aad588f52eb9a6, entries=150, sequenceid=192, filesize=11.9 K 2024-12-03T17:16:09,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/c38d66640f0643b0b51ce5dcc7adc7ec as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/c38d66640f0643b0b51ce5dcc7adc7ec 2024-12-03T17:16:09,391 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/c38d66640f0643b0b51ce5dcc7adc7ec, entries=150, sequenceid=192, filesize=11.9 K 2024-12-03T17:16:09,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/368af452cbc2404f9eff521d572442ac as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/368af452cbc2404f9eff521d572442ac 2024-12-03T17:16:09,416 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/368af452cbc2404f9eff521d572442ac, entries=150, sequenceid=192, filesize=11.9 K 2024-12-03T17:16:09,418 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for e1e0418f3d66f886e020279128ac6f55 in 1195ms, sequenceid=192, compaction requested=true 2024-12-03T17:16:09,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for e1e0418f3d66f886e020279128ac6f55: 2024-12-03T17:16:09,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:09,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-03T17:16:09,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-12-03T17:16:09,431 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-03T17:16:09,431 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7220 sec 2024-12-03T17:16:09,434 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.7420 sec 2024-12-03T17:16:09,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on e1e0418f3d66f886e020279128ac6f55 2024-12-03T17:16:09,462 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e1e0418f3d66f886e020279128ac6f55 3/3 column families, dataSize=134.18 KB heapSize=383.70 KB 2024-12-03T17:16:09,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=A 2024-12-03T17:16:09,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:09,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=B 2024-12-03T17:16:09,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:09,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=C 2024-12-03T17:16:09,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:09,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/9e75d5545a22417b9cf17962f248d7b5 is 50, key is test_row_0/A:col10/1733246168799/Put/seqid=0 2024-12-03T17:16:09,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:09,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47748 deadline: 1733246229488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:09,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:09,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:09,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:09,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:09,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1733246229488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:09,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:09,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:09,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:09,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:09,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246229508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:09,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:09,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:09,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:09,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:09,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246229512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:09,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:09,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246229519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:09,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741876_1052 (size=14541) 2024-12-03T17:16:09,555 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:09,555 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:09,555 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:09,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:09,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:09,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:09,557 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/9e75d5545a22417b9cf17962f248d7b5 2024-12-03T17:16:09,587 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/d25f24d105a5490da4c21ae4a8801300 is 50, key is test_row_0/B:col10/1733246168799/Put/seqid=0 2024-12-03T17:16:09,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:09,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1733246229624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:09,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:09,630 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:09,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47748 deadline: 1733246229626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:09,630 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:09,630 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:09,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:09,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:09,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:09,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741877_1053 (size=12151) 2024-12-03T17:16:09,646 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/d25f24d105a5490da4c21ae4a8801300 2024-12-03T17:16:09,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:09,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246229639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:09,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:09,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:09,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:09,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/4ba278af2cd8463eb9674c79b3592af4 is 50, key is test_row_0/C:col10/1733246168799/Put/seqid=0 2024-12-03T17:16:09,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:09,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246229665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:09,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:09,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:09,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:09,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:09,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246229665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:09,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:09,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:09,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:09,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741878_1054 (size=12151) 2024-12-03T17:16:09,723 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/4ba278af2cd8463eb9674c79b3592af4 2024-12-03T17:16:09,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/9e75d5545a22417b9cf17962f248d7b5 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/9e75d5545a22417b9cf17962f248d7b5 2024-12-03T17:16:09,804 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/9e75d5545a22417b9cf17962f248d7b5, entries=200, sequenceid=223, filesize=14.2 K 2024-12-03T17:16:09,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/d25f24d105a5490da4c21ae4a8801300 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/d25f24d105a5490da4c21ae4a8801300 2024-12-03T17:16:09,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:09,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1733246229835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:09,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:09,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:09,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:09,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:09,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47748 deadline: 1733246229836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:09,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:09,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:09,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:09,855 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/d25f24d105a5490da4c21ae4a8801300, entries=150, sequenceid=223, filesize=11.9 K 2024-12-03T17:16:09,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/4ba278af2cd8463eb9674c79b3592af4 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/4ba278af2cd8463eb9674c79b3592af4 2024-12-03T17:16:09,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/4ba278af2cd8463eb9674c79b3592af4, entries=150, sequenceid=223, filesize=11.9 K 2024-12-03T17:16:09,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T17:16:09,882 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:16:09,884 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~383.66 KB/392864, currentSize=46.96 KB/48090 for e1e0418f3d66f886e020279128ac6f55 in 421ms, sequenceid=223, compaction requested=true 2024-12-03T17:16:09,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e1e0418f3d66f886e020279128ac6f55: 2024-12-03T17:16:09,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1e0418f3d66f886e020279128ac6f55:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T17:16:09,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:09,884 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T17:16:09,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1e0418f3d66f886e020279128ac6f55:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T17:16:09,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:16:09,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1e0418f3d66f886e020279128ac6f55:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T17:16:09,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T17:16:09,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on e1e0418f3d66f886e020279128ac6f55 2024-12-03T17:16:09,886 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e1e0418f3d66f886e020279128ac6f55 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-03T17:16:09,886 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T17:16:09,886 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53726 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T17:16:09,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=A 2024-12-03T17:16:09,886 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): e1e0418f3d66f886e020279128ac6f55/A is initiating minor compaction (all files) 2024-12-03T17:16:09,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:09,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=B 2024-12-03T17:16:09,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:09,887 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1e0418f3d66f886e020279128ac6f55/A in TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:09,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=C 2024-12-03T17:16:09,887 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T17:16:09,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:09,887 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/7fd11e6dc0d145e09dcc71387c2afa6e, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/44db2c1cf0f046fbb654d96b2e6c2ea3, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/b9efbacfc85549de96aad588f52eb9a6, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/9e75d5545a22417b9cf17962f248d7b5] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp, totalSize=52.5 K 2024-12-03T17:16:09,891 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fd11e6dc0d145e09dcc71387c2afa6e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733246166918 2024-12-03T17:16:09,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-12-03T17:16:09,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-03T17:16:09,896 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T17:16:09,897 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T17:16:09,897 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T17:16:09,898 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 44db2c1cf0f046fbb654d96b2e6c2ea3, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733246167058 2024-12-03T17:16:09,901 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting b9efbacfc85549de96aad588f52eb9a6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733246167461 2024-12-03T17:16:09,902 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T17:16:09,902 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e75d5545a22417b9cf17962f248d7b5, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1733246168748 2024-12-03T17:16:09,902 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): e1e0418f3d66f886e020279128ac6f55/B is initiating minor compaction (all files) 2024-12-03T17:16:09,902 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1e0418f3d66f886e020279128ac6f55/B in TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:09,902 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/71278d223dc8459dacc71bb962c71dda, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/cb3f2576f3ad4faa8571ef8d07d51a82, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/c38d66640f0643b0b51ce5dcc7adc7ec, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/d25f24d105a5490da4c21ae4a8801300] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp, totalSize=47.8 K 2024-12-03T17:16:09,903 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71278d223dc8459dacc71bb962c71dda, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733246166918 2024-12-03T17:16:09,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/66d16e631d4948d6bf0461cbdf423918 is 50, key is test_row_0/A:col10/1733246169470/Put/seqid=0 2024-12-03T17:16:09,905 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb3f2576f3ad4faa8571ef8d07d51a82, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733246167058 2024-12-03T17:16:09,906 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting c38d66640f0643b0b51ce5dcc7adc7ec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733246167461 2024-12-03T17:16:09,908 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting d25f24d105a5490da4c21ae4a8801300, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1733246168748 2024-12-03T17:16:09,929 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1e0418f3d66f886e020279128ac6f55#A#compaction#43 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:16:09,931 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/2fc25b78dc7948a1831fece9a3eb7d7b is 50, key is test_row_0/A:col10/1733246168799/Put/seqid=0 2024-12-03T17:16:09,939 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1e0418f3d66f886e020279128ac6f55#B#compaction#44 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:16:09,940 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/bbf88b0ae55c4fc2a29eb540185268f8 is 50, key is test_row_0/B:col10/1733246168799/Put/seqid=0 2024-12-03T17:16:09,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741879_1055 (size=14541) 2024-12-03T17:16:09,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741880_1056 (size=12629) 2024-12-03T17:16:09,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741881_1057 (size=12629) 2024-12-03T17:16:10,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-03T17:16:10,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:10,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246230017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:10,038 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733246169971, max timestamp=1733246170033Num uniques 859; 2024-12-03T17:16:10,038 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733246169971, max timestamp=1733246170033Num uniques 859; 2024-12-03T17:16:10,038 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T17:16:10,038 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T17:16:10,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:10,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:10,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:10,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733246169971, max timestamp=1733246170033Num uniques 859; 2024-12-03T17:16:10,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T17:16:10,055 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-03T17:16:10,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:10,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. as already flushing 2024-12-03T17:16:10,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:10,055 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:10,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:10,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:10,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:10,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:10,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246230144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:10,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246230145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:10,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:10,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:10,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:10,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:10,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:10,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:10,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:10,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:10,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1733246230155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:10,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:10,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:10,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:10,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246230148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:10,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:10,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:10,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:10,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:10,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47748 deadline: 1733246230169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:10,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:10,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:10,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:10,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-03T17:16:10,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-03T17:16:10,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:10,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. as already flushing 2024-12-03T17:16:10,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:10,218 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:10,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:10,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:10,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:10,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246230265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:10,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:10,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:10,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:10,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:10,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246230275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:10,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:10,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:10,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:10,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/66d16e631d4948d6bf0461cbdf423918 2024-12-03T17:16:10,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:10,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246230365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:10,381 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:10,381 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:10,381 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:10,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-03T17:16:10,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:10,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. as already flushing 2024-12-03T17:16:10,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:10,382 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:10,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:10,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:10,399 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/eea341a160b24fe6851d4f59e7c4bf42 is 50, key is test_row_0/B:col10/1733246169470/Put/seqid=0 2024-12-03T17:16:10,419 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/2fc25b78dc7948a1831fece9a3eb7d7b as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/2fc25b78dc7948a1831fece9a3eb7d7b 2024-12-03T17:16:10,446 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/bbf88b0ae55c4fc2a29eb540185268f8 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/bbf88b0ae55c4fc2a29eb540185268f8 2024-12-03T17:16:10,448 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e1e0418f3d66f886e020279128ac6f55/A of e1e0418f3d66f886e020279128ac6f55 into 2fc25b78dc7948a1831fece9a3eb7d7b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:16:10,448 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1e0418f3d66f886e020279128ac6f55: 2024-12-03T17:16:10,448 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., storeName=e1e0418f3d66f886e020279128ac6f55/A, priority=12, startTime=1733246169884; duration=0sec 2024-12-03T17:16:10,449 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:16:10,449 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1e0418f3d66f886e020279128ac6f55:A 2024-12-03T17:16:10,449 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T17:16:10,457 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T17:16:10,458 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): e1e0418f3d66f886e020279128ac6f55/C is initiating minor compaction (all files) 2024-12-03T17:16:10,458 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1e0418f3d66f886e020279128ac6f55/C in TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:10,458 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/5e85c2c97bec403ca33238f586df759a, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/d49cc80422434fbaba448bdd5a1db9ce, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/368af452cbc2404f9eff521d572442ac, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/4ba278af2cd8463eb9674c79b3592af4] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp, totalSize=47.8 K 2024-12-03T17:16:10,459 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e85c2c97bec403ca33238f586df759a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733246166918 2024-12-03T17:16:10,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741882_1058 (size=12151) 2024-12-03T17:16:10,462 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting d49cc80422434fbaba448bdd5a1db9ce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733246167058 2024-12-03T17:16:10,463 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 368af452cbc2404f9eff521d572442ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733246167461 2024-12-03T17:16:10,465 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ba278af2cd8463eb9674c79b3592af4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1733246168748 2024-12-03T17:16:10,468 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e1e0418f3d66f886e020279128ac6f55/B of e1e0418f3d66f886e020279128ac6f55 into bbf88b0ae55c4fc2a29eb540185268f8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:16:10,468 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1e0418f3d66f886e020279128ac6f55: 2024-12-03T17:16:10,468 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., storeName=e1e0418f3d66f886e020279128ac6f55/B, priority=12, startTime=1733246169884; duration=0sec 2024-12-03T17:16:10,468 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:10,468 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1e0418f3d66f886e020279128ac6f55:B 2024-12-03T17:16:10,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:10,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246230487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:10,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:10,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:10,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:10,515 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1e0418f3d66f886e020279128ac6f55#C#compaction#46 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:16:10,516 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/99979cc602284e44a2e481af7458206f is 50, key is test_row_0/C:col10/1733246168799/Put/seqid=0 2024-12-03T17:16:10,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:10,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246230515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:10,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-03T17:16:10,526 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:10,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:10,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:10,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-03T17:16:10,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:10,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. as already flushing 2024-12-03T17:16:10,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:10,545 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:10,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:10,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:10,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741883_1059 (size=12629) 2024-12-03T17:16:10,599 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/99979cc602284e44a2e481af7458206f as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/99979cc602284e44a2e481af7458206f 2024-12-03T17:16:10,627 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e1e0418f3d66f886e020279128ac6f55/C of e1e0418f3d66f886e020279128ac6f55 into 99979cc602284e44a2e481af7458206f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:16:10,627 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1e0418f3d66f886e020279128ac6f55: 2024-12-03T17:16:10,627 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., storeName=e1e0418f3d66f886e020279128ac6f55/C, priority=12, startTime=1733246169884; duration=0sec 2024-12-03T17:16:10,627 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:10,627 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1e0418f3d66f886e020279128ac6f55:C 2024-12-03T17:16:10,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:10,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1733246230675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:10,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:10,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:10,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:10,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:10,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246230695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:10,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:10,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:10,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:10,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:10,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-03T17:16:10,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:10,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. as already flushing 2024-12-03T17:16:10,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47748 deadline: 1733246230699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:10,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:10,715 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:10,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:10,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:10,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:10,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:10,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:10,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:10,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246230804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:10,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:10,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:10,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:10,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:10,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246230838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:10,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:10,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:10,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:10,858 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/eea341a160b24fe6851d4f59e7c4bf42 2024-12-03T17:16:10,874 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-03T17:16:10,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:10,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. as already flushing 2024-12-03T17:16:10,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:10,875 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:10,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:10,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:10,925 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/81018ede9e344c808c42f3d28bee0e55 is 50, key is test_row_0/C:col10/1733246169470/Put/seqid=0 2024-12-03T17:16:10,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741884_1060 (size=12151) 2024-12-03T17:16:10,986 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/81018ede9e344c808c42f3d28bee0e55 2024-12-03T17:16:11,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/66d16e631d4948d6bf0461cbdf423918 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/66d16e631d4948d6bf0461cbdf423918 2024-12-03T17:16:11,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-03T17:16:11,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:11,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. as already flushing 2024-12-03T17:16:11,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:11,039 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:11,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:11,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:11,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-03T17:16:11,060 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/66d16e631d4948d6bf0461cbdf423918, entries=200, sequenceid=234, filesize=14.2 K 2024-12-03T17:16:11,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/eea341a160b24fe6851d4f59e7c4bf42 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/eea341a160b24fe6851d4f59e7c4bf42 2024-12-03T17:16:11,108 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/eea341a160b24fe6851d4f59e7c4bf42, entries=150, sequenceid=234, filesize=11.9 K 2024-12-03T17:16:11,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/81018ede9e344c808c42f3d28bee0e55 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/81018ede9e344c808c42f3d28bee0e55 2024-12-03T17:16:11,172 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/81018ede9e344c808c42f3d28bee0e55, entries=150, sequenceid=234, filesize=11.9 K 2024-12-03T17:16:11,178 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for e1e0418f3d66f886e020279128ac6f55 in 1287ms, sequenceid=234, compaction requested=false 2024-12-03T17:16:11,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e1e0418f3d66f886e020279128ac6f55: 2024-12-03T17:16:11,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-03T17:16:11,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:11,200 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing e1e0418f3d66f886e020279128ac6f55 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-12-03T17:16:11,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=A 2024-12-03T17:16:11,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:11,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=B 2024-12-03T17:16:11,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:11,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=C 2024-12-03T17:16:11,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:11,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. as already flushing 2024-12-03T17:16:11,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on e1e0418f3d66f886e020279128ac6f55 2024-12-03T17:16:11,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/be543f68e14841d482b7241190a05695 is 50, key is test_row_0/A:col10/1733246170145/Put/seqid=0 2024-12-03T17:16:11,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741885_1061 (size=12301) 2024-12-03T17:16:11,310 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/be543f68e14841d482b7241190a05695 2024-12-03T17:16:11,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:11,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246231324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:11,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:11,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:11,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:11,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:11,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246231336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:11,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:11,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:11,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:11,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/62ed8d718ee4487b9e7bfafbb9e18362 is 50, key is test_row_0/B:col10/1733246170145/Put/seqid=0 2024-12-03T17:16:11,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:11,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246231353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:11,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:11,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:11,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:11,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741886_1062 (size=12301) 2024-12-03T17:16:11,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:11,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246231454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:11,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:11,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:11,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:11,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:11,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246231674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:11,677 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:11,677 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:11,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:11,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:11,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1733246231698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:11,718 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 27e0f852a164,41167,1733246151593 for 'test_row_2' in e1e0418f3d66f886e020279128ac6f55 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T17:16:11,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:11,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:11,719 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:11,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:11,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47748 deadline: 1733246231726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:11,730 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 27e0f852a164,41167,1733246151593 for 'test_row_2' in e1e0418f3d66f886e020279128ac6f55 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2242 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T17:16:11,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:11,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:11,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:11,786 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/62ed8d718ee4487b9e7bfafbb9e18362 2024-12-03T17:16:11,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/8a444921bd07485489bc9803708b7baa is 50, key is test_row_0/C:col10/1733246170145/Put/seqid=0 2024-12-03T17:16:11,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741887_1063 (size=12301) 2024-12-03T17:16:11,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:11,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246231984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:11,994 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:11,994 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:11,994 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:12,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-03T17:16:12,258 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/8a444921bd07485489bc9803708b7baa 2024-12-03T17:16:12,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/be543f68e14841d482b7241190a05695 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/be543f68e14841d482b7241190a05695 2024-12-03T17:16:12,282 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/be543f68e14841d482b7241190a05695, entries=150, sequenceid=271, filesize=12.0 K 2024-12-03T17:16:12,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/62ed8d718ee4487b9e7bfafbb9e18362 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/62ed8d718ee4487b9e7bfafbb9e18362 2024-12-03T17:16:12,295 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/62ed8d718ee4487b9e7bfafbb9e18362, entries=150, sequenceid=271, filesize=12.0 K 2024-12-03T17:16:12,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/8a444921bd07485489bc9803708b7baa as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/8a444921bd07485489bc9803708b7baa 2024-12-03T17:16:12,314 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/8a444921bd07485489bc9803708b7baa, entries=150, sequenceid=271, filesize=12.0 K 2024-12-03T17:16:12,320 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~370.88 KB/379776, currentSize=53.67 KB/54960 for e1e0418f3d66f886e020279128ac6f55 in 1119ms, sequenceid=271, compaction requested=true 2024-12-03T17:16:12,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for e1e0418f3d66f886e020279128ac6f55: 2024-12-03T17:16:12,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:12,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-12-03T17:16:12,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-12-03T17:16:12,334 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-03T17:16:12,334 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4330 sec 2024-12-03T17:16:12,347 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 2.4500 sec 2024-12-03T17:16:12,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on e1e0418f3d66f886e020279128ac6f55 2024-12-03T17:16:12,359 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e1e0418f3d66f886e020279128ac6f55 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-03T17:16:12,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=A 2024-12-03T17:16:12,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:12,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=B 2024-12-03T17:16:12,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:12,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=C 2024-12-03T17:16:12,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:12,400 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/7fbe774e2ed04a7e8e7e4732845dd877 is 50, key is test_row_0/A:col10/1733246172355/Put/seqid=0 2024-12-03T17:16:12,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741888_1064 (size=12301) 2024-12-03T17:16:12,452 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/7fbe774e2ed04a7e8e7e4732845dd877 2024-12-03T17:16:12,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/20996568b25c4f8884c2157f1ed185ca is 50, key is test_row_0/B:col10/1733246172355/Put/seqid=0 2024-12-03T17:16:12,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741889_1065 (size=12301) 2024-12-03T17:16:12,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:12,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246232572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:12,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:12,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:12,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:12,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:12,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246232583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:12,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:12,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246232592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:12,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:12,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:12,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:12,603 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:12,603 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:12,603 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:12,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:12,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246232703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:12,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:12,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:12,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:12,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:12,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246232715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:12,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:12,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:12,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:12,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:12,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246232715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:12,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:12,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:12,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:12,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:12,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246232924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:12,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:12,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:12,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:12,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:12,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246232926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:12,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:12,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:12,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:12,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:12,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246232934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:12,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:12,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:12,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:12,943 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/20996568b25c4f8884c2157f1ed185ca 2024-12-03T17:16:12,968 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/4bdad1ba751748468532b4a6f0a44685 is 50, key is test_row_0/C:col10/1733246172355/Put/seqid=0 2024-12-03T17:16:13,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741890_1066 (size=12301) 2024-12-03T17:16:13,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:13,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246233233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:13,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:13,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:13,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:13,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:13,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246233244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:13,255 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:13,255 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:13,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:13,255 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:13,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246233247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:13,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:13,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:13,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:13,431 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/4bdad1ba751748468532b4a6f0a44685 2024-12-03T17:16:13,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/7fbe774e2ed04a7e8e7e4732845dd877 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/7fbe774e2ed04a7e8e7e4732845dd877 2024-12-03T17:16:13,608 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/7fbe774e2ed04a7e8e7e4732845dd877, entries=150, sequenceid=283, filesize=12.0 K 2024-12-03T17:16:13,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/20996568b25c4f8884c2157f1ed185ca as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/20996568b25c4f8884c2157f1ed185ca 2024-12-03T17:16:13,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/20996568b25c4f8884c2157f1ed185ca, entries=150, sequenceid=283, filesize=12.0 K 2024-12-03T17:16:13,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/4bdad1ba751748468532b4a6f0a44685 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/4bdad1ba751748468532b4a6f0a44685 2024-12-03T17:16:13,658 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/4bdad1ba751748468532b4a6f0a44685, entries=150, sequenceid=283, filesize=12.0 K 2024-12-03T17:16:13,660 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for e1e0418f3d66f886e020279128ac6f55 in 1301ms, sequenceid=283, compaction requested=true 2024-12-03T17:16:13,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e1e0418f3d66f886e020279128ac6f55: 2024-12-03T17:16:13,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1e0418f3d66f886e020279128ac6f55:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T17:16:13,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:13,661 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T17:16:13,661 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T17:16:13,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1e0418f3d66f886e020279128ac6f55:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T17:16:13,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:13,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1e0418f3d66f886e020279128ac6f55:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T17:16:13,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:16:13,668 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51772 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T17:16:13,668 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49382 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T17:16:13,668 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): e1e0418f3d66f886e020279128ac6f55/A is initiating minor compaction (all files) 2024-12-03T17:16:13,668 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): e1e0418f3d66f886e020279128ac6f55/B is initiating minor compaction (all files) 2024-12-03T17:16:13,668 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1e0418f3d66f886e020279128ac6f55/A in TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:13,669 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1e0418f3d66f886e020279128ac6f55/B in TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:13,669 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/2fc25b78dc7948a1831fece9a3eb7d7b, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/66d16e631d4948d6bf0461cbdf423918, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/be543f68e14841d482b7241190a05695, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/7fbe774e2ed04a7e8e7e4732845dd877] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp, totalSize=50.6 K 2024-12-03T17:16:13,669 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/bbf88b0ae55c4fc2a29eb540185268f8, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/eea341a160b24fe6851d4f59e7c4bf42, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/62ed8d718ee4487b9e7bfafbb9e18362, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/20996568b25c4f8884c2157f1ed185ca] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp, totalSize=48.2 K 2024-12-03T17:16:13,670 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting bbf88b0ae55c4fc2a29eb540185268f8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1733246168748 2024-12-03T17:16:13,670 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fc25b78dc7948a1831fece9a3eb7d7b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1733246168748 2024-12-03T17:16:13,678 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting eea341a160b24fe6851d4f59e7c4bf42, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733246169470 2024-12-03T17:16:13,682 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 62ed8d718ee4487b9e7bfafbb9e18362, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1733246170113 2024-12-03T17:16:13,682 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66d16e631d4948d6bf0461cbdf423918, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733246169470 2024-12-03T17:16:13,687 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting be543f68e14841d482b7241190a05695, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1733246170113 2024-12-03T17:16:13,688 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 20996568b25c4f8884c2157f1ed185ca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733246171229 2024-12-03T17:16:13,690 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fbe774e2ed04a7e8e7e4732845dd877, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733246171229 2024-12-03T17:16:13,740 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1e0418f3d66f886e020279128ac6f55#A#compaction#54 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:16:13,742 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1e0418f3d66f886e020279128ac6f55#B#compaction#55 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:16:13,743 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/c686a82cbf14495bbd6c65f8ad19e051 is 50, key is test_row_0/B:col10/1733246172355/Put/seqid=0 2024-12-03T17:16:13,744 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/8a4d58e9415f426eba9679739a513612 is 50, key is test_row_0/A:col10/1733246172355/Put/seqid=0 2024-12-03T17:16:13,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on e1e0418f3d66f886e020279128ac6f55 2024-12-03T17:16:13,747 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e1e0418f3d66f886e020279128ac6f55 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-03T17:16:13,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=A 2024-12-03T17:16:13,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:13,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=B 2024-12-03T17:16:13,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:13,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733246172524, max timestamp=1733246173737Num uniques 859; 2024-12-03T17:16:13,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T17:16:13,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=C 2024-12-03T17:16:13,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:13,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/f20c4160a7b34fd1b2b763e6081c2644 is 50, key is test_row_0/A:col10/1733246172570/Put/seqid=0 2024-12-03T17:16:13,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741891_1067 (size=12915) 2024-12-03T17:16:13,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741892_1068 (size=12915) 2024-12-03T17:16:13,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:13,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47748 deadline: 1733246233797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:13,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:13,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:13,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:13,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:13,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1733246233809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:13,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:13,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:13,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:13,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246233811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:13,816 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:13,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:13,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:13,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:13,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:13,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246233814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:13,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:13,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:13,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:13,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:13,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246233814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:13,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:13,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:13,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:13,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741893_1069 (size=12301) 2024-12-03T17:16:13,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:13,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:13,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1733246233928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:13,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47748 deadline: 1733246233926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:13,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:13,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:13,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:13,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:13,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246233929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:13,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:13,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:13,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:13,941 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:13,941 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:13,941 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:13,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:13,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246233936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:13,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:13,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:13,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:13,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:13,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246233936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:13,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:13,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:13,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:14,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-03T17:16:14,071 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:16:14,073 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T17:16:14,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-12-03T17:16:14,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-03T17:16:14,091 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T17:16:14,095 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T17:16:14,096 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T17:16:14,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:14,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1733246234144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:14,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:14,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:14,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:14,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:14,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246234144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:14,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:14,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:14,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:14,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:14,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246234156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:14,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:14,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246234156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:14,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:14,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:14,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:14,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:14,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:14,169 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:14,169 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:14,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47748 deadline: 1733246234156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:14,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:14,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:14,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:14,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-03T17:16:14,229 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/8a4d58e9415f426eba9679739a513612 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/8a4d58e9415f426eba9679739a513612 2024-12-03T17:16:14,230 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/c686a82cbf14495bbd6c65f8ad19e051 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/c686a82cbf14495bbd6c65f8ad19e051 2024-12-03T17:16:14,236 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/f20c4160a7b34fd1b2b763e6081c2644 2024-12-03T17:16:14,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-03T17:16:14,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:14,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. as already flushing 2024-12-03T17:16:14,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:14,263 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:14,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:14,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:14,271 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e1e0418f3d66f886e020279128ac6f55/B of e1e0418f3d66f886e020279128ac6f55 into c686a82cbf14495bbd6c65f8ad19e051(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:16:14,271 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e1e0418f3d66f886e020279128ac6f55/A of e1e0418f3d66f886e020279128ac6f55 into 8a4d58e9415f426eba9679739a513612(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:16:14,271 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1e0418f3d66f886e020279128ac6f55: 2024-12-03T17:16:14,271 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1e0418f3d66f886e020279128ac6f55: 2024-12-03T17:16:14,271 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., storeName=e1e0418f3d66f886e020279128ac6f55/B, priority=12, startTime=1733246173661; duration=0sec 2024-12-03T17:16:14,271 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., storeName=e1e0418f3d66f886e020279128ac6f55/A, priority=12, startTime=1733246173660; duration=0sec 2024-12-03T17:16:14,271 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:16:14,271 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1e0418f3d66f886e020279128ac6f55:B 2024-12-03T17:16:14,271 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T17:16:14,273 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:14,274 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1e0418f3d66f886e020279128ac6f55:A 2024-12-03T17:16:14,283 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49382 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T17:16:14,283 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): e1e0418f3d66f886e020279128ac6f55/C is initiating minor compaction (all files) 2024-12-03T17:16:14,284 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1e0418f3d66f886e020279128ac6f55/C in TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:14,284 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/99979cc602284e44a2e481af7458206f, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/81018ede9e344c808c42f3d28bee0e55, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/8a444921bd07485489bc9803708b7baa, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/4bdad1ba751748468532b4a6f0a44685] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp, totalSize=48.2 K 2024-12-03T17:16:14,288 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 99979cc602284e44a2e481af7458206f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1733246168748 2024-12-03T17:16:14,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/6f0a52c8d60143fba5314432a1b7671c is 50, key is test_row_0/B:col10/1733246172570/Put/seqid=0 2024-12-03T17:16:14,294 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 81018ede9e344c808c42f3d28bee0e55, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733246169470 2024-12-03T17:16:14,294 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a444921bd07485489bc9803708b7baa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1733246170113 2024-12-03T17:16:14,298 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bdad1ba751748468532b4a6f0a44685, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733246171229 2024-12-03T17:16:14,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741894_1070 (size=12301) 2024-12-03T17:16:14,342 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/6f0a52c8d60143fba5314432a1b7671c 2024-12-03T17:16:14,362 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1e0418f3d66f886e020279128ac6f55#C#compaction#58 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:16:14,363 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/bbf8aa28861d4948a804424f04094e49 is 50, key is test_row_0/C:col10/1733246172355/Put/seqid=0 2024-12-03T17:16:14,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-03T17:16:14,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741895_1071 (size=12915) 2024-12-03T17:16:14,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/15cfbdcee3f445d2b3bade3853416314 is 50, key is test_row_0/C:col10/1733246172570/Put/seqid=0 2024-12-03T17:16:14,428 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/bbf8aa28861d4948a804424f04094e49 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/bbf8aa28861d4948a804424f04094e49 2024-12-03T17:16:14,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-03T17:16:14,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:14,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. as already flushing 2024-12-03T17:16:14,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:14,438 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:14,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:14,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:14,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741896_1072 (size=12301) 2024-12-03T17:16:14,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:14,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1733246234454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:14,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:14,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:14,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:14,463 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e1e0418f3d66f886e020279128ac6f55/C of e1e0418f3d66f886e020279128ac6f55 into bbf8aa28861d4948a804424f04094e49(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:16:14,463 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1e0418f3d66f886e020279128ac6f55: 2024-12-03T17:16:14,463 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., storeName=e1e0418f3d66f886e020279128ac6f55/C, priority=12, startTime=1733246173662; duration=0sec 2024-12-03T17:16:14,463 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:14,463 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1e0418f3d66f886e020279128ac6f55:C 2024-12-03T17:16:14,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:14,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246234464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:14,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:14,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:14,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:14,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:14,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47748 deadline: 1733246234476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:14,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:14,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246234476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:14,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:14,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:14,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:14,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:14,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246234476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:14,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:14,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:14,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:14,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:14,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:14,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:14,603 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-03T17:16:14,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:14,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. as already flushing 2024-12-03T17:16:14,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:14,604 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:14,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:14,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:14,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-03T17:16:14,763 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-03T17:16:14,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:14,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. as already flushing 2024-12-03T17:16:14,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:14,764 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:14,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:14,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:14,849 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/15cfbdcee3f445d2b3bade3853416314 2024-12-03T17:16:14,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/f20c4160a7b34fd1b2b763e6081c2644 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/f20c4160a7b34fd1b2b763e6081c2644 2024-12-03T17:16:14,907 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/f20c4160a7b34fd1b2b763e6081c2644, entries=150, sequenceid=304, filesize=12.0 K 2024-12-03T17:16:14,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/6f0a52c8d60143fba5314432a1b7671c as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/6f0a52c8d60143fba5314432a1b7671c 2024-12-03T17:16:14,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-03T17:16:14,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:14,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. as already flushing 2024-12-03T17:16:14,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:14,925 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:14,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:14,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:14,961 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/6f0a52c8d60143fba5314432a1b7671c, entries=150, sequenceid=304, filesize=12.0 K 2024-12-03T17:16:14,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/15cfbdcee3f445d2b3bade3853416314 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/15cfbdcee3f445d2b3bade3853416314 2024-12-03T17:16:14,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:14,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1733246234974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:14,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:14,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:14,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:15,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:15,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47748 deadline: 1733246234994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:15,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:14,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:15,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246234994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:15,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:15,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:15,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246234995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:15,005 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:15,005 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:15,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:15,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:15,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:15,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:15,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:15,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:15,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246234994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:15,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:15,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:15,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:15,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/15cfbdcee3f445d2b3bade3853416314, entries=150, sequenceid=304, filesize=12.0 K 2024-12-03T17:16:15,032 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~89.45 KB/91600, heapSize ~262.06 KB/268352, currentSize=87.22 KB/89310 for e1e0418f3d66f886e020279128ac6f55 in 1285ms, sequenceid=304, compaction requested=false 2024-12-03T17:16:15,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e1e0418f3d66f886e020279128ac6f55: 2024-12-03T17:16:15,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-03T17:16:15,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:15,095 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing e1e0418f3d66f886e020279128ac6f55 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-03T17:16:15,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=A 2024-12-03T17:16:15,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:15,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=B 2024-12-03T17:16:15,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:15,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=C 2024-12-03T17:16:15,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:15,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/bb46d8ae9a084261911778a8b11c6128 is 50, key is test_row_0/A:col10/1733246173792/Put/seqid=0 2024-12-03T17:16:15,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741897_1073 (size=12301) 2024-12-03T17:16:15,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-03T17:16:15,589 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/bb46d8ae9a084261911778a8b11c6128 2024-12-03T17:16:15,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/6d737715420946a8a0bad9ef742592b1 is 50, key is test_row_0/B:col10/1733246173792/Put/seqid=0 2024-12-03T17:16:15,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741898_1074 (size=12301) 2024-12-03T17:16:16,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. as already flushing 2024-12-03T17:16:16,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on e1e0418f3d66f886e020279128ac6f55 2024-12-03T17:16:16,051 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/6d737715420946a8a0bad9ef742592b1 2024-12-03T17:16:16,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/1f8f4990b247488089431b0c23a9598a is 50, key is test_row_0/C:col10/1733246173792/Put/seqid=0 2024-12-03T17:16:16,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:16,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1733246236073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:16,084 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:16,084 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:16,084 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:16,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:16,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246236076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:16,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:16,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:16,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:16,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:16,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47748 deadline: 1733246236080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:16,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:16,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246236084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:16,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:16,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:16,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:16,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:16,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:16,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:16,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:16,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246236088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:16,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:16,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:16,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:16,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741899_1075 (size=12301) 2024-12-03T17:16:16,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:16,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:16,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246236201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:16,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246236201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:16,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:16,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:16,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1733246236202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:16,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47748 deadline: 1733246236201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:16,214 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:16,214 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:16,214 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:16,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:16,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:16,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:16,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:16,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246236203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:16,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:16,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:16,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:16,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:16,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:16,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:16,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:16,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:16,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:16,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-03T17:16:16,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:16,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246236427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:16,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:16,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:16,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:16,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:16,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:16,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246236428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:16,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47748 deadline: 1733246236427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:16,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:16,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:16,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:16,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:16,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:16,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1733246236428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:16,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:16,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:16,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:16,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246236428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:16,436 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:16,436 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:16,436 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:16,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:16,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:16,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:16,518 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/1f8f4990b247488089431b0c23a9598a 2024-12-03T17:16:16,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/bb46d8ae9a084261911778a8b11c6128 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/bb46d8ae9a084261911778a8b11c6128 2024-12-03T17:16:16,586 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/bb46d8ae9a084261911778a8b11c6128, entries=150, sequenceid=323, filesize=12.0 K 2024-12-03T17:16:16,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/6d737715420946a8a0bad9ef742592b1 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/6d737715420946a8a0bad9ef742592b1 2024-12-03T17:16:16,602 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/6d737715420946a8a0bad9ef742592b1, entries=150, sequenceid=323, filesize=12.0 K 2024-12-03T17:16:16,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/1f8f4990b247488089431b0c23a9598a as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/1f8f4990b247488089431b0c23a9598a 2024-12-03T17:16:16,615 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/1f8f4990b247488089431b0c23a9598a, entries=150, sequenceid=323, filesize=12.0 K 2024-12-03T17:16:16,617 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for e1e0418f3d66f886e020279128ac6f55 in 1522ms, sequenceid=323, compaction requested=true 2024-12-03T17:16:16,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for e1e0418f3d66f886e020279128ac6f55: 2024-12-03T17:16:16,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:16,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-12-03T17:16:16,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-12-03T17:16:16,640 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-03T17:16:16,641 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5320 sec 2024-12-03T17:16:16,645 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 2.5690 sec 2024-12-03T17:16:16,748 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e1e0418f3d66f886e020279128ac6f55 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-03T17:16:16,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on e1e0418f3d66f886e020279128ac6f55 2024-12-03T17:16:16,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=A 2024-12-03T17:16:16,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:16,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=B 2024-12-03T17:16:16,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:16,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=C 2024-12-03T17:16:16,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:16,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/8faa922dcd3d41669fc13a0cd84d0849 is 50, key is test_row_0/A:col10/1733246176746/Put/seqid=0 2024-12-03T17:16:16,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:16,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246236767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:16,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:16,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:16,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:16,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:16,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1733246236774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:16,781 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:16,781 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:16,781 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:16,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:16,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:16,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47748 deadline: 1733246236779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:16,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246236779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:16,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:16,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:16,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:16,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:16,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246236783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:16,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:16,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:16,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:16,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:16,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:16,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:16,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741900_1076 (size=12301) 2024-12-03T17:16:16,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:16,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246236883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:16,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:16,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:16,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1733246236884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:16,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:16,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:16,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:16,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:16,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:16,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:16,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47748 deadline: 1733246236904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:16,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:16,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246236904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:16,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:16,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246236904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:16,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:16,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:16,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:16,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:16,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:16,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:16,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:16,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:16,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:17,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:17,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246237094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:17,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:17,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:17,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:17,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:17,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1733246237094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:17,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:17,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:17,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:17,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:17,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246237114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:17,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:17,123 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:17,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246237114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:17,123 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:17,123 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:17,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:17,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:17,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:17,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:17,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47748 deadline: 1733246237114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:17,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:17,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:17,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:17,213 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/8faa922dcd3d41669fc13a0cd84d0849 2024-12-03T17:16:17,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/572dc83e1a6f49929cd21e2911131dbd is 50, key is test_row_0/B:col10/1733246176746/Put/seqid=0 2024-12-03T17:16:17,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741901_1077 (size=12301) 2024-12-03T17:16:17,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:17,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246237403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:17,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:17,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:17,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:17,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:17,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1733246237414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:17,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:17,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:17,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:17,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:17,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47748 deadline: 1733246237435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:17,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:17,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246237435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:17,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:17,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:17,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:17,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:17,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:17,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:17,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:17,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246237435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:17,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:17,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:17,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:17,699 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/572dc83e1a6f49929cd21e2911131dbd 2024-12-03T17:16:17,744 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/f52a22219b9146afb180323e92ff4afc is 50, key is test_row_0/C:col10/1733246176746/Put/seqid=0 2024-12-03T17:16:17,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741902_1078 (size=12301) 2024-12-03T17:16:17,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:17,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246237914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:17,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:17,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:17,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:17,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:17,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1733246237924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:17,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:17,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:17,931 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:17,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:17,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47748 deadline: 1733246237954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:17,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:17,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246237954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:17,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:17,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246237955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:17,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:17,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:17,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:17,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:17,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:17,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:17,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:17,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:17,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:18,185 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/f52a22219b9146afb180323e92ff4afc 2024-12-03T17:16:18,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/8faa922dcd3d41669fc13a0cd84d0849 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/8faa922dcd3d41669fc13a0cd84d0849 2024-12-03T17:16:18,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/8faa922dcd3d41669fc13a0cd84d0849, entries=150, sequenceid=343, filesize=12.0 K 2024-12-03T17:16:18,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/572dc83e1a6f49929cd21e2911131dbd as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/572dc83e1a6f49929cd21e2911131dbd 2024-12-03T17:16:18,241 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/572dc83e1a6f49929cd21e2911131dbd, entries=150, sequenceid=343, filesize=12.0 K 2024-12-03T17:16:18,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/f52a22219b9146afb180323e92ff4afc as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/f52a22219b9146afb180323e92ff4afc 2024-12-03T17:16:18,265 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/f52a22219b9146afb180323e92ff4afc, entries=150, sequenceid=343, filesize=12.0 K 2024-12-03T17:16:18,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-03T17:16:18,268 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for e1e0418f3d66f886e020279128ac6f55 in 1519ms, sequenceid=343, compaction requested=true 2024-12-03T17:16:18,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e1e0418f3d66f886e020279128ac6f55: 2024-12-03T17:16:18,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1e0418f3d66f886e020279128ac6f55:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T17:16:18,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:18,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1e0418f3d66f886e020279128ac6f55:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T17:16:18,268 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T17:16:18,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:16:18,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1e0418f3d66f886e020279128ac6f55:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T17:16:18,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T17:16:18,269 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T17:16:18,271 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:16:18,275 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T17:16:18,276 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T17:16:18,276 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): e1e0418f3d66f886e020279128ac6f55/A is initiating minor compaction (all files) 2024-12-03T17:16:18,276 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1e0418f3d66f886e020279128ac6f55/A in TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:18,276 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/8a4d58e9415f426eba9679739a513612, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/f20c4160a7b34fd1b2b763e6081c2644, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/bb46d8ae9a084261911778a8b11c6128, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/8faa922dcd3d41669fc13a0cd84d0849] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp, totalSize=48.7 K 2024-12-03T17:16:18,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-12-03T17:16:18,279 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T17:16:18,279 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): e1e0418f3d66f886e020279128ac6f55/B is initiating minor compaction (all files) 2024-12-03T17:16:18,279 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1e0418f3d66f886e020279128ac6f55/B in TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:18,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T17:16:18,279 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/c686a82cbf14495bbd6c65f8ad19e051, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/6f0a52c8d60143fba5314432a1b7671c, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/6d737715420946a8a0bad9ef742592b1, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/572dc83e1a6f49929cd21e2911131dbd] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp, totalSize=48.7 K 2024-12-03T17:16:18,280 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a4d58e9415f426eba9679739a513612, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733246171229 2024-12-03T17:16:18,280 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T17:16:18,282 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T17:16:18,283 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T17:16:18,283 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting c686a82cbf14495bbd6c65f8ad19e051, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733246171229 2024-12-03T17:16:18,285 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting f20c4160a7b34fd1b2b763e6081c2644, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1733246172570 2024-12-03T17:16:18,285 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f0a52c8d60143fba5314432a1b7671c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1733246172570 2024-12-03T17:16:18,288 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting bb46d8ae9a084261911778a8b11c6128, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1733246173792 2024-12-03T17:16:18,289 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d737715420946a8a0bad9ef742592b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1733246173792 2024-12-03T17:16:18,289 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 8faa922dcd3d41669fc13a0cd84d0849, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1733246176077 2024-12-03T17:16:18,290 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 572dc83e1a6f49929cd21e2911131dbd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1733246176077 2024-12-03T17:16:18,342 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1e0418f3d66f886e020279128ac6f55#A#compaction#66 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:16:18,343 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/f0a0bf93e38c47bca36cf291dc63e08f is 50, key is test_row_0/A:col10/1733246176746/Put/seqid=0 2024-12-03T17:16:18,354 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1e0418f3d66f886e020279128ac6f55#B#compaction#67 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:16:18,355 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/f465b72bcdb0477aa81748c7cd9badf9 is 50, key is test_row_0/B:col10/1733246176746/Put/seqid=0 2024-12-03T17:16:18,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T17:16:18,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741903_1079 (size=13051) 2024-12-03T17:16:18,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741904_1080 (size=13051) 2024-12-03T17:16:18,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-03T17:16:18,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:18,438 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing e1e0418f3d66f886e020279128ac6f55 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T17:16:18,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=A 2024-12-03T17:16:18,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:18,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=B 2024-12-03T17:16:18,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:18,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=C 2024-12-03T17:16:18,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:18,447 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/f465b72bcdb0477aa81748c7cd9badf9 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/f465b72bcdb0477aa81748c7cd9badf9 2024-12-03T17:16:18,467 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/f0a0bf93e38c47bca36cf291dc63e08f as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/f0a0bf93e38c47bca36cf291dc63e08f 2024-12-03T17:16:18,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/249fa6db7f154d7197b37c95cf854d4e is 50, key is test_row_0/A:col10/1733246176769/Put/seqid=0 2024-12-03T17:16:18,491 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e1e0418f3d66f886e020279128ac6f55/B of e1e0418f3d66f886e020279128ac6f55 into f465b72bcdb0477aa81748c7cd9badf9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:16:18,491 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1e0418f3d66f886e020279128ac6f55: 2024-12-03T17:16:18,491 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., storeName=e1e0418f3d66f886e020279128ac6f55/B, priority=12, startTime=1733246178268; duration=0sec 2024-12-03T17:16:18,492 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:16:18,492 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1e0418f3d66f886e020279128ac6f55:B 2024-12-03T17:16:18,492 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T17:16:18,497 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T17:16:18,497 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): e1e0418f3d66f886e020279128ac6f55/C is initiating minor compaction (all files) 2024-12-03T17:16:18,497 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1e0418f3d66f886e020279128ac6f55/C in TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:18,497 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/bbf8aa28861d4948a804424f04094e49, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/15cfbdcee3f445d2b3bade3853416314, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/1f8f4990b247488089431b0c23a9598a, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/f52a22219b9146afb180323e92ff4afc] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp, totalSize=48.7 K 2024-12-03T17:16:18,502 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbf8aa28861d4948a804424f04094e49, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1733246171229 2024-12-03T17:16:18,503 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15cfbdcee3f445d2b3bade3853416314, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1733246172570 2024-12-03T17:16:18,511 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f8f4990b247488089431b0c23a9598a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1733246173792 2024-12-03T17:16:18,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741905_1081 (size=12301) 2024-12-03T17:16:18,519 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting f52a22219b9146afb180323e92ff4afc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1733246176077 2024-12-03T17:16:18,535 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e1e0418f3d66f886e020279128ac6f55/A of e1e0418f3d66f886e020279128ac6f55 into f0a0bf93e38c47bca36cf291dc63e08f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:16:18,535 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1e0418f3d66f886e020279128ac6f55: 2024-12-03T17:16:18,536 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., storeName=e1e0418f3d66f886e020279128ac6f55/A, priority=12, startTime=1733246178268; duration=0sec 2024-12-03T17:16:18,536 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:18,536 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1e0418f3d66f886e020279128ac6f55:A 2024-12-03T17:16:18,575 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1e0418f3d66f886e020279128ac6f55#C#compaction#69 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:16:18,576 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/9015705aebb54a46aba4302185668727 is 50, key is test_row_0/C:col10/1733246176746/Put/seqid=0 2024-12-03T17:16:18,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T17:16:18,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741906_1082 (size=13051) 2024-12-03T17:16:18,647 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/9015705aebb54a46aba4302185668727 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/9015705aebb54a46aba4302185668727 2024-12-03T17:16:18,660 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e1e0418f3d66f886e020279128ac6f55/C of e1e0418f3d66f886e020279128ac6f55 into 9015705aebb54a46aba4302185668727(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:16:18,660 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1e0418f3d66f886e020279128ac6f55: 2024-12-03T17:16:18,660 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., storeName=e1e0418f3d66f886e020279128ac6f55/C, priority=12, startTime=1733246178268; duration=0sec 2024-12-03T17:16:18,660 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:18,660 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1e0418f3d66f886e020279128ac6f55:C 2024-12-03T17:16:18,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T17:16:18,922 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/249fa6db7f154d7197b37c95cf854d4e 2024-12-03T17:16:18,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on e1e0418f3d66f886e020279128ac6f55 2024-12-03T17:16:18,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. as already flushing 2024-12-03T17:16:19,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/134210b2b69840b5bc4184b10254ff00 is 50, key is test_row_0/B:col10/1733246176769/Put/seqid=0 2024-12-03T17:16:19,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741907_1083 (size=12301) 2024-12-03T17:16:19,065 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/134210b2b69840b5bc4184b10254ff00 2024-12-03T17:16:19,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:19,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47748 deadline: 1733246239067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:19,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:19,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:19,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:19,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733246179055, max timestamp=1733246179092Num uniques 859; 2024-12-03T17:16:19,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T17:16:19,109 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733246179055, max timestamp=1733246179092Num uniques 859; 2024-12-03T17:16:19,109 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T17:16:19,116 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733246179055, max timestamp=1733246179092Num uniques 859; 2024-12-03T17:16:19,116 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T17:16:19,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/4622a1754b3e416abf0d5871b9e3e5cb is 50, key is test_row_0/C:col10/1733246176769/Put/seqid=0 2024-12-03T17:16:19,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741908_1084 (size=12301) 2024-12-03T17:16:19,166 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-12-03T17:16:19,166 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:16:19,168 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-12-03T17:16:19,168 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:16:19,171 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-12-03T17:16:19,171 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:16:19,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:19,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246239154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:19,172 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:19,172 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:19,172 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:19,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:19,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246239157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:19,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:19,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:19,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:19,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:19,177 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:19,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246239153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:19,177 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:19,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:19,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:19,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:19,178 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:16:19,178 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:19,178 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:19,178 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:19,178 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:16:19,179 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:19,179 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:16:19,182 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-12-03T17:16:19,182 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:16:19,182 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:19,182 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:19,182 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:16:19,188 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-12-03T17:16:19,188 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:16:19,188 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:19,188 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:19,189 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:16:19,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:19,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47748 deadline: 1733246239210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:19,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:19,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:19,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:19,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:19,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246239285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:19,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:19,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246239285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:19,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:19,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246239285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:19,286 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:19,286 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:19,286 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:19,287 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:19,287 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:19,287 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:19,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:19,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:19,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:19,399 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T17:16:19,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T17:16:19,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:19,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47748 deadline: 1733246239424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:19,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:19,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:19,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:19,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:19,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:19,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47738 deadline: 1733246239494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:19,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47768 deadline: 1733246239494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:19,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:19,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47750 deadline: 1733246239494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:19,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:19,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:19,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:19,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:19,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:19,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:19,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:19,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:19,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:19,566 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/4622a1754b3e416abf0d5871b9e3e5cb 2024-12-03T17:16:19,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/249fa6db7f154d7197b37c95cf854d4e as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/249fa6db7f154d7197b37c95cf854d4e 2024-12-03T17:16:19,593 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/249fa6db7f154d7197b37c95cf854d4e, entries=150, sequenceid=356, filesize=12.0 K 2024-12-03T17:16:19,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/134210b2b69840b5bc4184b10254ff00 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/134210b2b69840b5bc4184b10254ff00 2024-12-03T17:16:19,636 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/134210b2b69840b5bc4184b10254ff00, entries=150, sequenceid=356, filesize=12.0 K 2024-12-03T17:16:19,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/4622a1754b3e416abf0d5871b9e3e5cb as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/4622a1754b3e416abf0d5871b9e3e5cb 2024-12-03T17:16:19,655 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/4622a1754b3e416abf0d5871b9e3e5cb, entries=150, sequenceid=356, filesize=12.0 K 2024-12-03T17:16:19,656 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for e1e0418f3d66f886e020279128ac6f55 in 1218ms, sequenceid=356, compaction requested=false 2024-12-03T17:16:19,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for e1e0418f3d66f886e020279128ac6f55: 2024-12-03T17:16:19,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:19,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-12-03T17:16:19,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-12-03T17:16:19,663 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-03T17:16:19,663 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3770 sec 2024-12-03T17:16:19,668 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.3900 sec 2024-12-03T17:16:19,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on e1e0418f3d66f886e020279128ac6f55 2024-12-03T17:16:19,738 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e1e0418f3d66f886e020279128ac6f55 3/3 column families, dataSize=120.76 KB heapSize=345.14 KB 2024-12-03T17:16:19,738 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-12-03T17:16:19,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=A 2024-12-03T17:16:19,738 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:16:19,738 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:19,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:19,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=B 2024-12-03T17:16:19,739 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:19,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:19,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=C 2024-12-03T17:16:19,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:19,739 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:16:19,760 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/8633945261724a8c86d628ec5da1ebc2 is 50, key is test_row_0/A:col10/1733246179152/Put/seqid=0 2024-12-03T17:16:19,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741909_1085 (size=12301) 2024-12-03T17:16:19,806 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-12-03T17:16:19,806 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:16:19,806 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:19,806 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:19,806 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:16:19,813 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-12-03T17:16:19,814 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:16:19,814 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:19,814 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:19,814 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:16:19,814 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-12-03T17:16:19,814 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:16:19,814 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:19,815 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:19,815 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:16:20,194 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/8633945261724a8c86d628ec5da1ebc2 2024-12-03T17:16:20,205 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/5cfa85fa764d4c068199e21bd0bbfedf is 50, key is test_row_0/B:col10/1733246179152/Put/seqid=0 2024-12-03T17:16:20,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741910_1086 (size=12301) 2024-12-03T17:16:20,210 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/5cfa85fa764d4c068199e21bd0bbfedf 2024-12-03T17:16:20,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/42ebb4f13bf045cea5fb4c9a5f198bc5 is 50, key is test_row_0/C:col10/1733246179152/Put/seqid=0 2024-12-03T17:16:20,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741911_1087 (size=12301) 2024-12-03T17:16:20,232 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/42ebb4f13bf045cea5fb4c9a5f198bc5 2024-12-03T17:16:20,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/8633945261724a8c86d628ec5da1ebc2 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/8633945261724a8c86d628ec5da1ebc2 2024-12-03T17:16:20,249 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/8633945261724a8c86d628ec5da1ebc2, entries=150, sequenceid=394, filesize=12.0 K 2024-12-03T17:16:20,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/5cfa85fa764d4c068199e21bd0bbfedf as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/5cfa85fa764d4c068199e21bd0bbfedf 2024-12-03T17:16:20,262 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/5cfa85fa764d4c068199e21bd0bbfedf, entries=150, sequenceid=394, filesize=12.0 K 2024-12-03T17:16:20,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/42ebb4f13bf045cea5fb4c9a5f198bc5 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/42ebb4f13bf045cea5fb4c9a5f198bc5 2024-12-03T17:16:20,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/42ebb4f13bf045cea5fb4c9a5f198bc5, entries=150, sequenceid=394, filesize=12.0 K 2024-12-03T17:16:20,274 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~345.09 KB/353376, currentSize=20.13 KB/20610 for e1e0418f3d66f886e020279128ac6f55 in 536ms, sequenceid=394, compaction requested=true 2024-12-03T17:16:20,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e1e0418f3d66f886e020279128ac6f55: 2024-12-03T17:16:20,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1e0418f3d66f886e020279128ac6f55:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T17:16:20,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:20,274 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:16:20,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1e0418f3d66f886e020279128ac6f55:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T17:16:20,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:20,274 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:16:20,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1e0418f3d66f886e020279128ac6f55:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T17:16:20,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:16:20,275 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:16:20,275 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:16:20,276 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): e1e0418f3d66f886e020279128ac6f55/A is initiating minor compaction (all files) 2024-12-03T17:16:20,276 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): e1e0418f3d66f886e020279128ac6f55/B is initiating minor compaction (all files) 2024-12-03T17:16:20,276 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1e0418f3d66f886e020279128ac6f55/A in TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:20,276 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1e0418f3d66f886e020279128ac6f55/B in TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:20,276 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/f0a0bf93e38c47bca36cf291dc63e08f, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/249fa6db7f154d7197b37c95cf854d4e, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/8633945261724a8c86d628ec5da1ebc2] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp, totalSize=36.8 K 2024-12-03T17:16:20,276 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/f465b72bcdb0477aa81748c7cd9badf9, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/134210b2b69840b5bc4184b10254ff00, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/5cfa85fa764d4c068199e21bd0bbfedf] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp, totalSize=36.8 K 2024-12-03T17:16:20,276 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0a0bf93e38c47bca36cf291dc63e08f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1733246176077 2024-12-03T17:16:20,276 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting f465b72bcdb0477aa81748c7cd9badf9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1733246176077 2024-12-03T17:16:20,277 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 134210b2b69840b5bc4184b10254ff00, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1733246176769 2024-12-03T17:16:20,277 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 249fa6db7f154d7197b37c95cf854d4e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1733246176769 2024-12-03T17:16:20,278 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cfa85fa764d4c068199e21bd0bbfedf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1733246179152 2024-12-03T17:16:20,278 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8633945261724a8c86d628ec5da1ebc2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1733246179152 2024-12-03T17:16:20,322 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1e0418f3d66f886e020279128ac6f55#A#compaction#75 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:16:20,323 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/3716e4c08b7e4e5298f82b35076f71d5 is 50, key is test_row_0/A:col10/1733246179152/Put/seqid=0 2024-12-03T17:16:20,345 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1e0418f3d66f886e020279128ac6f55#B#compaction#76 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:16:20,352 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/3d9cc316da474da5a4f4208353dc9b76 is 50, key is test_row_0/B:col10/1733246179152/Put/seqid=0 2024-12-03T17:16:20,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741912_1088 (size=13153) 2024-12-03T17:16:20,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741913_1089 (size=13153) 2024-12-03T17:16:20,419 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/3716e4c08b7e4e5298f82b35076f71d5 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/3716e4c08b7e4e5298f82b35076f71d5 2024-12-03T17:16:20,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T17:16:20,440 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:16:20,441 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-03T17:16:20,441 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 74 2024-12-03T17:16:20,441 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-12-03T17:16:20,441 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-12-03T17:16:20,441 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-12-03T17:16:20,441 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-12-03T17:16:20,441 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-03T17:16:20,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1280 2024-12-03T17:16:20,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1267 2024-12-03T17:16:20,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-03T17:16:20,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 642 2024-12-03T17:16:20,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1925 rows 2024-12-03T17:16:20,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 641 2024-12-03T17:16:20,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1922 rows 2024-12-03T17:16:20,442 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T17:16:20,442 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T17:16:20,442 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:20,443 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:20,443 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:16:20,456 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-03T17:16:20,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-03T17:16:20,477 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e1e0418f3d66f886e020279128ac6f55/A of e1e0418f3d66f886e020279128ac6f55 into 3716e4c08b7e4e5298f82b35076f71d5(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:16:20,477 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1e0418f3d66f886e020279128ac6f55: 2024-12-03T17:16:20,477 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., storeName=e1e0418f3d66f886e020279128ac6f55/A, priority=13, startTime=1733246180274; duration=0sec 2024-12-03T17:16:20,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-03T17:16:20,477 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:16:20,477 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1e0418f3d66f886e020279128ac6f55:A 2024-12-03T17:16:20,477 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:16:20,478 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/3d9cc316da474da5a4f4208353dc9b76 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/3d9cc316da474da5a4f4208353dc9b76 2024-12-03T17:16:20,483 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733246180483"}]},"ts":"1733246180483"} 2024-12-03T17:16:20,489 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:16:20,489 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): e1e0418f3d66f886e020279128ac6f55/C is initiating minor compaction (all files) 2024-12-03T17:16:20,489 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1e0418f3d66f886e020279128ac6f55/C in TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:20,489 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/9015705aebb54a46aba4302185668727, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/4622a1754b3e416abf0d5871b9e3e5cb, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/42ebb4f13bf045cea5fb4c9a5f198bc5] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp, totalSize=36.8 K 2024-12-03T17:16:20,494 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9015705aebb54a46aba4302185668727, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1733246176077 2024-12-03T17:16:20,497 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-03T17:16:20,497 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-03T17:16:20,498 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4622a1754b3e416abf0d5871b9e3e5cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1733246176769 2024-12-03T17:16:20,499 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42ebb4f13bf045cea5fb4c9a5f198bc5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1733246179152 2024-12-03T17:16:20,500 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T17:16:20,508 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e1e0418f3d66f886e020279128ac6f55, UNASSIGN}] 2024-12-03T17:16:20,512 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e1e0418f3d66f886e020279128ac6f55, UNASSIGN 2024-12-03T17:16:20,515 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=e1e0418f3d66f886e020279128ac6f55, regionState=CLOSING, regionLocation=27e0f852a164,41167,1733246151593 2024-12-03T17:16:20,520 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e1e0418f3d66f886e020279128ac6f55, UNASSIGN because future has completed 2024-12-03T17:16:20,521 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T17:16:20,521 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; CloseRegionProcedure e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593}] 2024-12-03T17:16:20,528 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e1e0418f3d66f886e020279128ac6f55/B of e1e0418f3d66f886e020279128ac6f55 into 3d9cc316da474da5a4f4208353dc9b76(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:16:20,528 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1e0418f3d66f886e020279128ac6f55: 2024-12-03T17:16:20,528 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., storeName=e1e0418f3d66f886e020279128ac6f55/B, priority=13, startTime=1733246180274; duration=0sec 2024-12-03T17:16:20,528 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:20,528 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1e0418f3d66f886e020279128ac6f55:B 2024-12-03T17:16:20,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-03T17:16:20,602 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1e0418f3d66f886e020279128ac6f55#C#compaction#77 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:16:20,609 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/d7536a2502a34b588d634a490f7f4507 is 50, key is test_row_0/C:col10/1733246179152/Put/seqid=0 2024-12-03T17:16:20,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741914_1090 (size=13153) 2024-12-03T17:16:20,697 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] handler.UnassignRegionHandler(122): Close e1e0418f3d66f886e020279128ac6f55 2024-12-03T17:16:20,697 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T17:16:20,698 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1722): Closing e1e0418f3d66f886e020279128ac6f55, disabling compactions & flushes 2024-12-03T17:16:20,698 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1993): waiting for 1 compactions to complete for region TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:20,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-03T17:16:21,042 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/d7536a2502a34b588d634a490f7f4507 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/d7536a2502a34b588d634a490f7f4507 2024-12-03T17:16:21,051 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e1e0418f3d66f886e020279128ac6f55/C of e1e0418f3d66f886e020279128ac6f55 into d7536a2502a34b588d634a490f7f4507(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:16:21,051 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1e0418f3d66f886e020279128ac6f55: 2024-12-03T17:16:21,051 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:21,052 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., storeName=e1e0418f3d66f886e020279128ac6f55/C, priority=13, startTime=1733246180274; duration=0sec 2024-12-03T17:16:21,052 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:21,052 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:21,052 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. after waiting 0 ms 2024-12-03T17:16:21,052 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1e0418f3d66f886e020279128ac6f55:C 2024-12-03T17:16:21,052 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:21,052 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(2902): Flushing e1e0418f3d66f886e020279128ac6f55 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-12-03T17:16:21,052 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=A 2024-12-03T17:16:21,052 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:21,052 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=B 2024-12-03T17:16:21,052 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:21,052 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1e0418f3d66f886e020279128ac6f55, store=C 2024-12-03T17:16:21,053 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:21,058 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/4a14a3dd85294999a91509b89f9e1bcc is 50, key is test_row_0/A:col10/1733246179807/Put/seqid=0 2024-12-03T17:16:21,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741915_1091 (size=9857) 2024-12-03T17:16:21,084 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/4a14a3dd85294999a91509b89f9e1bcc 2024-12-03T17:16:21,096 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/6762ecfbe1e147cdae93b1a16d8410ec is 50, key is test_row_0/B:col10/1733246179807/Put/seqid=0 2024-12-03T17:16:21,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-03T17:16:21,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741916_1092 (size=9857) 2024-12-03T17:16:21,133 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/6762ecfbe1e147cdae93b1a16d8410ec 2024-12-03T17:16:21,152 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/d1aa29124cbd47529fbed0dfb617e808 is 50, key is test_row_0/C:col10/1733246179807/Put/seqid=0 2024-12-03T17:16:21,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741917_1093 (size=9857) 2024-12-03T17:16:21,603 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/d1aa29124cbd47529fbed0dfb617e808 2024-12-03T17:16:21,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-03T17:16:21,634 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/A/4a14a3dd85294999a91509b89f9e1bcc as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/4a14a3dd85294999a91509b89f9e1bcc 2024-12-03T17:16:21,647 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/4a14a3dd85294999a91509b89f9e1bcc, entries=100, sequenceid=403, filesize=9.6 K 2024-12-03T17:16:21,649 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/B/6762ecfbe1e147cdae93b1a16d8410ec as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/6762ecfbe1e147cdae93b1a16d8410ec 2024-12-03T17:16:21,662 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/6762ecfbe1e147cdae93b1a16d8410ec, entries=100, sequenceid=403, filesize=9.6 K 2024-12-03T17:16:21,671 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/.tmp/C/d1aa29124cbd47529fbed0dfb617e808 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/d1aa29124cbd47529fbed0dfb617e808 2024-12-03T17:16:21,729 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/d1aa29124cbd47529fbed0dfb617e808, entries=100, sequenceid=403, filesize=9.6 K 2024-12-03T17:16:21,742 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for e1e0418f3d66f886e020279128ac6f55 in 690ms, sequenceid=403, compaction requested=false 2024-12-03T17:16:21,751 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/c65d6c13ed1942c6b9c430a51b7f0c40, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/914e83bbeaf14ece9ed23e921dbae1ca, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/f045a0c070c740e680e22bbafdd66410, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/9eadc9e9251a4849ab6d32e94d96af73, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/1615876ac85c4b0492db1b1ea2e5e2d4, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/9f3f9df63327452da33b9bcd020d6a29, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/cdbf3bd3ce2c432bbbe1722d094f228b, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/f4e6db0ca4604b388836071c0cce301e, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/0758a8578dda425f94814140139181d3, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/4c94eb44dfef4b7fac4092c4fc03f77a, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/7fd11e6dc0d145e09dcc71387c2afa6e, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/44db2c1cf0f046fbb654d96b2e6c2ea3, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/b9efbacfc85549de96aad588f52eb9a6, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/9e75d5545a22417b9cf17962f248d7b5, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/2fc25b78dc7948a1831fece9a3eb7d7b, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/66d16e631d4948d6bf0461cbdf423918, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/be543f68e14841d482b7241190a05695, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/8a4d58e9415f426eba9679739a513612, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/7fbe774e2ed04a7e8e7e4732845dd877, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/f20c4160a7b34fd1b2b763e6081c2644, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/bb46d8ae9a084261911778a8b11c6128, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/f0a0bf93e38c47bca36cf291dc63e08f, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/8faa922dcd3d41669fc13a0cd84d0849, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/249fa6db7f154d7197b37c95cf854d4e, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/8633945261724a8c86d628ec5da1ebc2] to archive 2024-12-03T17:16:21,755 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T17:16:21,763 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/c65d6c13ed1942c6b9c430a51b7f0c40 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/c65d6c13ed1942c6b9c430a51b7f0c40 2024-12-03T17:16:21,769 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/914e83bbeaf14ece9ed23e921dbae1ca to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/914e83bbeaf14ece9ed23e921dbae1ca 2024-12-03T17:16:21,774 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/f045a0c070c740e680e22bbafdd66410 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/f045a0c070c740e680e22bbafdd66410 2024-12-03T17:16:21,778 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/9eadc9e9251a4849ab6d32e94d96af73 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/9eadc9e9251a4849ab6d32e94d96af73 2024-12-03T17:16:21,790 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/1615876ac85c4b0492db1b1ea2e5e2d4 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/1615876ac85c4b0492db1b1ea2e5e2d4 2024-12-03T17:16:21,811 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/9f3f9df63327452da33b9bcd020d6a29 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/9f3f9df63327452da33b9bcd020d6a29 2024-12-03T17:16:21,823 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/cdbf3bd3ce2c432bbbe1722d094f228b to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/cdbf3bd3ce2c432bbbe1722d094f228b 2024-12-03T17:16:21,840 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/f4e6db0ca4604b388836071c0cce301e to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/f4e6db0ca4604b388836071c0cce301e 2024-12-03T17:16:21,844 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/0758a8578dda425f94814140139181d3 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/0758a8578dda425f94814140139181d3 2024-12-03T17:16:21,848 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/4c94eb44dfef4b7fac4092c4fc03f77a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/4c94eb44dfef4b7fac4092c4fc03f77a 2024-12-03T17:16:21,851 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/7fd11e6dc0d145e09dcc71387c2afa6e to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/7fd11e6dc0d145e09dcc71387c2afa6e 2024-12-03T17:16:21,858 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/44db2c1cf0f046fbb654d96b2e6c2ea3 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/44db2c1cf0f046fbb654d96b2e6c2ea3 2024-12-03T17:16:21,870 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/b9efbacfc85549de96aad588f52eb9a6 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/b9efbacfc85549de96aad588f52eb9a6 2024-12-03T17:16:21,882 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/9e75d5545a22417b9cf17962f248d7b5 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/9e75d5545a22417b9cf17962f248d7b5 2024-12-03T17:16:21,894 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/2fc25b78dc7948a1831fece9a3eb7d7b to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/2fc25b78dc7948a1831fece9a3eb7d7b 2024-12-03T17:16:21,911 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/66d16e631d4948d6bf0461cbdf423918 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/66d16e631d4948d6bf0461cbdf423918 2024-12-03T17:16:21,914 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/be543f68e14841d482b7241190a05695 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/be543f68e14841d482b7241190a05695 2024-12-03T17:16:21,918 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/8a4d58e9415f426eba9679739a513612 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/8a4d58e9415f426eba9679739a513612 2024-12-03T17:16:21,922 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/7fbe774e2ed04a7e8e7e4732845dd877 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/7fbe774e2ed04a7e8e7e4732845dd877 2024-12-03T17:16:21,926 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/f20c4160a7b34fd1b2b763e6081c2644 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/f20c4160a7b34fd1b2b763e6081c2644 2024-12-03T17:16:21,934 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/bb46d8ae9a084261911778a8b11c6128 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/bb46d8ae9a084261911778a8b11c6128 2024-12-03T17:16:21,946 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/f0a0bf93e38c47bca36cf291dc63e08f to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/f0a0bf93e38c47bca36cf291dc63e08f 2024-12-03T17:16:21,960 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/8faa922dcd3d41669fc13a0cd84d0849 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/8faa922dcd3d41669fc13a0cd84d0849 2024-12-03T17:16:21,963 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/249fa6db7f154d7197b37c95cf854d4e to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/249fa6db7f154d7197b37c95cf854d4e 2024-12-03T17:16:21,966 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/8633945261724a8c86d628ec5da1ebc2 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/8633945261724a8c86d628ec5da1ebc2 2024-12-03T17:16:22,001 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/a38c5bd6befd44b8b5e0591b4b5b0c83, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/afc44c870ad044539c4a37a880dd36e1, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/f356195a9ecb4d4eb307520735db42a5, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/3093491aa9b74728abe470d574c6cd09, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/f321d13903b24e37a1e51de1fc4866e1, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/d773d21cc5ee4e9d9db9641dd7c2bbb4, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/e683ac9734f74f5f98a549bb0961b0b2, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/b78e406e719649338ba157358c80f6e7, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/402986351b3c4b02a960ea59c0006f53, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/71278d223dc8459dacc71bb962c71dda, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/696fff161e5f41a58746827b44d9d9a5, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/cb3f2576f3ad4faa8571ef8d07d51a82, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/c38d66640f0643b0b51ce5dcc7adc7ec, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/bbf88b0ae55c4fc2a29eb540185268f8, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/d25f24d105a5490da4c21ae4a8801300, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/eea341a160b24fe6851d4f59e7c4bf42, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/62ed8d718ee4487b9e7bfafbb9e18362, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/c686a82cbf14495bbd6c65f8ad19e051, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/20996568b25c4f8884c2157f1ed185ca, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/6f0a52c8d60143fba5314432a1b7671c, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/6d737715420946a8a0bad9ef742592b1, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/f465b72bcdb0477aa81748c7cd9badf9, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/572dc83e1a6f49929cd21e2911131dbd, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/134210b2b69840b5bc4184b10254ff00, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/5cfa85fa764d4c068199e21bd0bbfedf] to archive 2024-12-03T17:16:22,004 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T17:16:22,011 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/a38c5bd6befd44b8b5e0591b4b5b0c83 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/a38c5bd6befd44b8b5e0591b4b5b0c83 2024-12-03T17:16:22,022 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/afc44c870ad044539c4a37a880dd36e1 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/afc44c870ad044539c4a37a880dd36e1 2024-12-03T17:16:22,030 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/f356195a9ecb4d4eb307520735db42a5 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/f356195a9ecb4d4eb307520735db42a5 2024-12-03T17:16:22,039 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/3093491aa9b74728abe470d574c6cd09 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/3093491aa9b74728abe470d574c6cd09 2024-12-03T17:16:22,041 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/f321d13903b24e37a1e51de1fc4866e1 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/f321d13903b24e37a1e51de1fc4866e1 2024-12-03T17:16:22,043 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/d773d21cc5ee4e9d9db9641dd7c2bbb4 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/d773d21cc5ee4e9d9db9641dd7c2bbb4 2024-12-03T17:16:22,044 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/e683ac9734f74f5f98a549bb0961b0b2 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/e683ac9734f74f5f98a549bb0961b0b2 2024-12-03T17:16:22,046 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/b78e406e719649338ba157358c80f6e7 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/b78e406e719649338ba157358c80f6e7 2024-12-03T17:16:22,055 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/402986351b3c4b02a960ea59c0006f53 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/402986351b3c4b02a960ea59c0006f53 2024-12-03T17:16:22,057 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/71278d223dc8459dacc71bb962c71dda to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/71278d223dc8459dacc71bb962c71dda 2024-12-03T17:16:22,059 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/696fff161e5f41a58746827b44d9d9a5 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/696fff161e5f41a58746827b44d9d9a5 2024-12-03T17:16:22,060 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/cb3f2576f3ad4faa8571ef8d07d51a82 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/cb3f2576f3ad4faa8571ef8d07d51a82 2024-12-03T17:16:22,062 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/c38d66640f0643b0b51ce5dcc7adc7ec to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/c38d66640f0643b0b51ce5dcc7adc7ec 2024-12-03T17:16:22,064 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/bbf88b0ae55c4fc2a29eb540185268f8 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/bbf88b0ae55c4fc2a29eb540185268f8 2024-12-03T17:16:22,066 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/d25f24d105a5490da4c21ae4a8801300 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/d25f24d105a5490da4c21ae4a8801300 2024-12-03T17:16:22,070 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/eea341a160b24fe6851d4f59e7c4bf42 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/eea341a160b24fe6851d4f59e7c4bf42 2024-12-03T17:16:22,072 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/62ed8d718ee4487b9e7bfafbb9e18362 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/62ed8d718ee4487b9e7bfafbb9e18362 2024-12-03T17:16:22,074 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/c686a82cbf14495bbd6c65f8ad19e051 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/c686a82cbf14495bbd6c65f8ad19e051 2024-12-03T17:16:22,076 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/20996568b25c4f8884c2157f1ed185ca to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/20996568b25c4f8884c2157f1ed185ca 2024-12-03T17:16:22,080 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/6f0a52c8d60143fba5314432a1b7671c to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/6f0a52c8d60143fba5314432a1b7671c 2024-12-03T17:16:22,083 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/6d737715420946a8a0bad9ef742592b1 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/6d737715420946a8a0bad9ef742592b1 2024-12-03T17:16:22,086 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/f465b72bcdb0477aa81748c7cd9badf9 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/f465b72bcdb0477aa81748c7cd9badf9 2024-12-03T17:16:22,089 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/572dc83e1a6f49929cd21e2911131dbd to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/572dc83e1a6f49929cd21e2911131dbd 2024-12-03T17:16:22,091 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/134210b2b69840b5bc4184b10254ff00 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/134210b2b69840b5bc4184b10254ff00 2024-12-03T17:16:22,093 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/5cfa85fa764d4c068199e21bd0bbfedf to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/5cfa85fa764d4c068199e21bd0bbfedf 2024-12-03T17:16:22,111 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/3077a4922e89443abbf0eca785510bf2, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/1da0d6b3fff94ce08ed729056a9c4901, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/52dc01b1874e4939b1bab5303281d6a6, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/682fc2860e8744f28de2f3b1d5629dbf, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/4327c46957304386915c494d6aa01609, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/402a04bfd59247d396c26265a5744bca, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/76d9ffdc5eaa41d1b6e921ba50ccb617, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/065813df88d6491bb396a00c79d9aa44, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/1d1f4a0bae6d4630a88665261a971701, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/5e85c2c97bec403ca33238f586df759a, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/d60c16f0d1ee49479791200b763e456a, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/d49cc80422434fbaba448bdd5a1db9ce, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/368af452cbc2404f9eff521d572442ac, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/99979cc602284e44a2e481af7458206f, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/4ba278af2cd8463eb9674c79b3592af4, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/81018ede9e344c808c42f3d28bee0e55, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/8a444921bd07485489bc9803708b7baa, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/bbf8aa28861d4948a804424f04094e49, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/4bdad1ba751748468532b4a6f0a44685, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/15cfbdcee3f445d2b3bade3853416314, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/1f8f4990b247488089431b0c23a9598a, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/9015705aebb54a46aba4302185668727, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/f52a22219b9146afb180323e92ff4afc, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/4622a1754b3e416abf0d5871b9e3e5cb, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/42ebb4f13bf045cea5fb4c9a5f198bc5] to archive 2024-12-03T17:16:22,118 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T17:16:22,136 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/3077a4922e89443abbf0eca785510bf2 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/3077a4922e89443abbf0eca785510bf2 2024-12-03T17:16:22,144 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/1da0d6b3fff94ce08ed729056a9c4901 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/1da0d6b3fff94ce08ed729056a9c4901 2024-12-03T17:16:22,151 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/52dc01b1874e4939b1bab5303281d6a6 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/52dc01b1874e4939b1bab5303281d6a6 2024-12-03T17:16:22,161 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/682fc2860e8744f28de2f3b1d5629dbf to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/682fc2860e8744f28de2f3b1d5629dbf 2024-12-03T17:16:22,163 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/4327c46957304386915c494d6aa01609 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/4327c46957304386915c494d6aa01609 2024-12-03T17:16:22,165 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/402a04bfd59247d396c26265a5744bca to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/402a04bfd59247d396c26265a5744bca 2024-12-03T17:16:22,171 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/76d9ffdc5eaa41d1b6e921ba50ccb617 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/76d9ffdc5eaa41d1b6e921ba50ccb617 2024-12-03T17:16:22,187 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/065813df88d6491bb396a00c79d9aa44 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/065813df88d6491bb396a00c79d9aa44 2024-12-03T17:16:22,213 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/1d1f4a0bae6d4630a88665261a971701 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/1d1f4a0bae6d4630a88665261a971701 2024-12-03T17:16:22,217 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/5e85c2c97bec403ca33238f586df759a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/5e85c2c97bec403ca33238f586df759a 2024-12-03T17:16:22,218 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/d60c16f0d1ee49479791200b763e456a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/d60c16f0d1ee49479791200b763e456a 2024-12-03T17:16:22,230 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/d49cc80422434fbaba448bdd5a1db9ce to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/d49cc80422434fbaba448bdd5a1db9ce 2024-12-03T17:16:22,233 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/368af452cbc2404f9eff521d572442ac to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/368af452cbc2404f9eff521d572442ac 2024-12-03T17:16:22,237 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/99979cc602284e44a2e481af7458206f to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/99979cc602284e44a2e481af7458206f 2024-12-03T17:16:22,242 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/4ba278af2cd8463eb9674c79b3592af4 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/4ba278af2cd8463eb9674c79b3592af4 2024-12-03T17:16:22,246 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/81018ede9e344c808c42f3d28bee0e55 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/81018ede9e344c808c42f3d28bee0e55 2024-12-03T17:16:22,249 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/8a444921bd07485489bc9803708b7baa to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/8a444921bd07485489bc9803708b7baa 2024-12-03T17:16:22,253 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/bbf8aa28861d4948a804424f04094e49 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/bbf8aa28861d4948a804424f04094e49 2024-12-03T17:16:22,256 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/4bdad1ba751748468532b4a6f0a44685 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/4bdad1ba751748468532b4a6f0a44685 2024-12-03T17:16:22,258 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/15cfbdcee3f445d2b3bade3853416314 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/15cfbdcee3f445d2b3bade3853416314 2024-12-03T17:16:22,261 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/1f8f4990b247488089431b0c23a9598a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/1f8f4990b247488089431b0c23a9598a 2024-12-03T17:16:22,263 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/9015705aebb54a46aba4302185668727 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/9015705aebb54a46aba4302185668727 2024-12-03T17:16:22,265 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/f52a22219b9146afb180323e92ff4afc to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/f52a22219b9146afb180323e92ff4afc 2024-12-03T17:16:22,266 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/4622a1754b3e416abf0d5871b9e3e5cb to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/4622a1754b3e416abf0d5871b9e3e5cb 2024-12-03T17:16:22,268 DEBUG [StoreCloser-TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/42ebb4f13bf045cea5fb4c9a5f198bc5 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/42ebb4f13bf045cea5fb4c9a5f198bc5 2024-12-03T17:16:22,294 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/recovered.edits/406.seqid, newMaxSeqId=406, maxSeqId=1 2024-12-03T17:16:22,295 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. 2024-12-03T17:16:22,296 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1676): Region close journal for e1e0418f3d66f886e020279128ac6f55: Waiting for close lock at 1733246180698Running coprocessor pre-close hooks at 1733246180698Disabling compacts and flushes for region at 1733246180698Disabling writes for close at 1733246181052 (+354 ms)Obtaining lock to block concurrent updates at 1733246181052Preparing flush snapshotting stores in e1e0418f3d66f886e020279128ac6f55 at 1733246181052Finished memstore snapshotting TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1733246181053 (+1 ms)Flushing stores of TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55. at 1733246181053Flushing e1e0418f3d66f886e020279128ac6f55/A: creating writer at 1733246181053Flushing e1e0418f3d66f886e020279128ac6f55/A: appending metadata at 1733246181057 (+4 ms)Flushing e1e0418f3d66f886e020279128ac6f55/A: closing flushed file at 1733246181057Flushing e1e0418f3d66f886e020279128ac6f55/B: creating writer at 1733246181092 (+35 ms)Flushing e1e0418f3d66f886e020279128ac6f55/B: appending metadata at 1733246181096 (+4 ms)Flushing e1e0418f3d66f886e020279128ac6f55/B: closing flushed file at 1733246181096Flushing e1e0418f3d66f886e020279128ac6f55/C: creating writer at 1733246181142 (+46 ms)Flushing e1e0418f3d66f886e020279128ac6f55/C: appending metadata at 1733246181151 (+9 ms)Flushing e1e0418f3d66f886e020279128ac6f55/C: closing flushed file at 1733246181151Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1eb5f3d5: reopening flushed file at 1733246181628 (+477 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@23d85d58: reopening flushed file at 1733246181647 (+19 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@31677094: reopening flushed file at 1733246181663 (+16 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for e1e0418f3d66f886e020279128ac6f55 in 690ms, sequenceid=403, compaction requested=false at 1733246181742 (+79 ms)Writing region close event to WAL at 1733246182283 (+541 ms)Running coprocessor post-close hooks at 1733246182295 (+12 ms)Closed at 1733246182295 2024-12-03T17:16:22,305 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] handler.UnassignRegionHandler(157): Closed e1e0418f3d66f886e020279128ac6f55 2024-12-03T17:16:22,308 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=e1e0418f3d66f886e020279128ac6f55, regionState=CLOSED 2024-12-03T17:16:22,316 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; CloseRegionProcedure e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 because future has completed 2024-12-03T17:16:22,322 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-03T17:16:22,322 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; CloseRegionProcedure e1e0418f3d66f886e020279128ac6f55, server=27e0f852a164,41167,1733246151593 in 1.7990 sec 2024-12-03T17:16:22,325 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-12-03T17:16:22,326 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e1e0418f3d66f886e020279128ac6f55, UNASSIGN in 1.8140 sec 2024-12-03T17:16:22,329 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-12-03T17:16:22,329 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8270 sec 2024-12-03T17:16:22,331 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733246182330"}]},"ts":"1733246182330"} 2024-12-03T17:16:22,333 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-03T17:16:22,333 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-03T17:16:22,336 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8750 sec 2024-12-03T17:16:22,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-03T17:16:22,635 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:16:22,640 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-03T17:16:22,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T17:16:22,648 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=25, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T17:16:22,651 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=25, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T17:16:22,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-03T17:16:22,671 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55 2024-12-03T17:16:22,685 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A, FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B, FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C, FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/recovered.edits] 2024-12-03T17:16:22,691 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/3716e4c08b7e4e5298f82b35076f71d5 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/3716e4c08b7e4e5298f82b35076f71d5 2024-12-03T17:16:22,694 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/4a14a3dd85294999a91509b89f9e1bcc to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/A/4a14a3dd85294999a91509b89f9e1bcc 2024-12-03T17:16:22,700 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/3d9cc316da474da5a4f4208353dc9b76 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/3d9cc316da474da5a4f4208353dc9b76 2024-12-03T17:16:22,702 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/6762ecfbe1e147cdae93b1a16d8410ec to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/B/6762ecfbe1e147cdae93b1a16d8410ec 2024-12-03T17:16:22,711 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/d1aa29124cbd47529fbed0dfb617e808 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/d1aa29124cbd47529fbed0dfb617e808 2024-12-03T17:16:22,714 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/d7536a2502a34b588d634a490f7f4507 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/C/d7536a2502a34b588d634a490f7f4507 2024-12-03T17:16:22,722 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/recovered.edits/406.seqid to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55/recovered.edits/406.seqid 2024-12-03T17:16:22,723 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/e1e0418f3d66f886e020279128ac6f55 2024-12-03T17:16:22,723 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-03T17:16:22,735 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=25, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T17:16:22,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-12-03T17:16:22,754 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-03T17:16:22,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-03T17:16:22,801 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-03T17:16:22,803 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=25, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T17:16:22,804 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-03T17:16:22,804 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733246182804"}]},"ts":"9223372036854775807"} 2024-12-03T17:16:22,810 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-03T17:16:22,810 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e1e0418f3d66f886e020279128ac6f55, NAME => 'TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55.', STARTKEY => '', ENDKEY => ''}] 2024-12-03T17:16:22,811 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-03T17:16:22,811 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733246182811"}]},"ts":"9223372036854775807"} 2024-12-03T17:16:22,815 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-03T17:16:22,817 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=25, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T17:16:22,820 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 176 msec 2024-12-03T17:16:22,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-03T17:16:22,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-03T17:16:22,976 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:16:23,000 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=249 (was 210) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_782644355_22 at /127.0.0.1:38988 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1405682164_22 at /127.0.0.1:49548 [Waiting for operation #133] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1405682164_22 at /127.0.0.1:47614 [Waiting for operation #188] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/27e0f852a164:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS:0;27e0f852a164:41167-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_782644355_22 at /127.0.0.1:49436 [Waiting for operation #176] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1483 (was 1114) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4728 (was 6385) 2024-12-03T17:16:23,017 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=249, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=1483, ProcessCount=11, AvailableMemoryMB=4726 2024-12-03T17:16:23,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T17:16:23,023 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T17:16:23,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-03T17:16:23,034 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T17:16:23,034 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:23,035 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 26 2024-12-03T17:16:23,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-12-03T17:16:23,041 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T17:16:23,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741918_1094 (size=1061) 2024-12-03T17:16:23,106 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4 2024-12-03T17:16:23,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-12-03T17:16:23,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741919_1095 (size=53) 2024-12-03T17:16:23,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-12-03T17:16:23,552 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T17:16:23,552 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 4e02d2d095200ac1f1a0b05c28cb936a, disabling compactions & flushes 2024-12-03T17:16:23,552 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:23,552 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:23,552 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. after waiting 0 ms 2024-12-03T17:16:23,552 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:23,552 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:23,552 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4e02d2d095200ac1f1a0b05c28cb936a: Waiting for close lock at 1733246183552Disabling compacts and flushes for region at 1733246183552Disabling writes for close at 1733246183552Writing region close event to WAL at 1733246183552Closed at 1733246183552 2024-12-03T17:16:23,554 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T17:16:23,555 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733246183554"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733246183554"}]},"ts":"1733246183554"} 2024-12-03T17:16:23,558 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T17:16:23,559 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T17:16:23,560 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733246183559"}]},"ts":"1733246183559"} 2024-12-03T17:16:23,562 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-03T17:16:23,563 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4e02d2d095200ac1f1a0b05c28cb936a, ASSIGN}] 2024-12-03T17:16:23,565 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4e02d2d095200ac1f1a0b05c28cb936a, ASSIGN 2024-12-03T17:16:23,567 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4e02d2d095200ac1f1a0b05c28cb936a, ASSIGN; state=OFFLINE, location=27e0f852a164,41167,1733246151593; forceNewPlan=false, retain=false 2024-12-03T17:16:23,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-12-03T17:16:23,718 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=4e02d2d095200ac1f1a0b05c28cb936a, regionState=OPENING, regionLocation=27e0f852a164,41167,1733246151593 2024-12-03T17:16:23,720 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4e02d2d095200ac1f1a0b05c28cb936a, ASSIGN because future has completed 2024-12-03T17:16:23,721 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593}] 2024-12-03T17:16:23,880 INFO [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:23,880 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7752): Opening region: {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} 2024-12-03T17:16:23,881 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:23,881 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T17:16:23,881 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7794): checking encryption for 4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:23,881 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7797): checking classloading for 4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:23,884 INFO [StoreOpener-4e02d2d095200ac1f1a0b05c28cb936a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:23,887 INFO [StoreOpener-4e02d2d095200ac1f1a0b05c28cb936a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T17:16:23,888 INFO [StoreOpener-4e02d2d095200ac1f1a0b05c28cb936a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4e02d2d095200ac1f1a0b05c28cb936a columnFamilyName A 2024-12-03T17:16:23,888 DEBUG [StoreOpener-4e02d2d095200ac1f1a0b05c28cb936a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:23,889 INFO [StoreOpener-4e02d2d095200ac1f1a0b05c28cb936a-1 {}] regionserver.HStore(327): Store=4e02d2d095200ac1f1a0b05c28cb936a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T17:16:23,889 INFO [StoreOpener-4e02d2d095200ac1f1a0b05c28cb936a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:23,891 INFO [StoreOpener-4e02d2d095200ac1f1a0b05c28cb936a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T17:16:23,891 INFO [StoreOpener-4e02d2d095200ac1f1a0b05c28cb936a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4e02d2d095200ac1f1a0b05c28cb936a columnFamilyName B 2024-12-03T17:16:23,891 DEBUG [StoreOpener-4e02d2d095200ac1f1a0b05c28cb936a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:23,892 INFO [StoreOpener-4e02d2d095200ac1f1a0b05c28cb936a-1 {}] regionserver.HStore(327): Store=4e02d2d095200ac1f1a0b05c28cb936a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T17:16:23,892 INFO [StoreOpener-4e02d2d095200ac1f1a0b05c28cb936a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:23,895 INFO [StoreOpener-4e02d2d095200ac1f1a0b05c28cb936a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T17:16:23,895 INFO [StoreOpener-4e02d2d095200ac1f1a0b05c28cb936a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4e02d2d095200ac1f1a0b05c28cb936a columnFamilyName C 2024-12-03T17:16:23,895 DEBUG [StoreOpener-4e02d2d095200ac1f1a0b05c28cb936a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:23,896 INFO [StoreOpener-4e02d2d095200ac1f1a0b05c28cb936a-1 {}] regionserver.HStore(327): Store=4e02d2d095200ac1f1a0b05c28cb936a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T17:16:23,896 INFO [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:23,896 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1038): replaying wal for 4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:23,897 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:23,897 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:23,900 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1048): stopping wal replay for 4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:23,900 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1060): Cleaning up temporary data for 4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:23,900 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T17:16:23,903 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): writing seq id for 4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:23,909 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T17:16:23,918 INFO [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1114): Opened 4e02d2d095200ac1f1a0b05c28cb936a; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59254881, jitterRate=-0.1170334666967392}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T17:16:23,918 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:23,920 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1006): Region open journal for 4e02d2d095200ac1f1a0b05c28cb936a: Running coprocessor pre-open hook at 1733246183882Writing region info on filesystem at 1733246183882Initializing all the Stores at 1733246183883 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733246183883Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733246183884 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733246183884Cleaning up temporary data from old regions at 1733246183900 (+16 ms)Running coprocessor post-open hooks at 1733246183918 (+18 ms)Region opened successfully at 1733246183920 (+2 ms) 2024-12-03T17:16:23,934 INFO [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., pid=28, masterSystemTime=1733246183874 2024-12-03T17:16:23,942 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:23,942 INFO [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:23,942 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=4e02d2d095200ac1f1a0b05c28cb936a, regionState=OPEN, openSeqNum=2, regionLocation=27e0f852a164,41167,1733246151593 2024-12-03T17:16:23,952 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 because future has completed 2024-12-03T17:16:23,960 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-12-03T17:16:23,960 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 in 235 msec 2024-12-03T17:16:23,967 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-12-03T17:16:23,967 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4e02d2d095200ac1f1a0b05c28cb936a, ASSIGN in 398 msec 2024-12-03T17:16:23,971 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T17:16:23,972 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733246183971"}]},"ts":"1733246183971"} 2024-12-03T17:16:23,981 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-03T17:16:23,983 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T17:16:23,987 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 959 msec 2024-12-03T17:16:24,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-12-03T17:16:24,175 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:16:24,183 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b4db6c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:24,183 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:16:24,183 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:16:24,186 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:16:24,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:16:24,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:16:24,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4810677, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:24,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:16:24,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:16:24,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:24,194 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43372, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:16:24,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f00002a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:24,199 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:16:24,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:16:24,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:16:24,206 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38944, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:16:24,209 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T17:16:24,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 27e0f852a164,38523,1733246150595 2024-12-03T17:16:24,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@427eb97b 2024-12-03T17:16:24,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T17:16:24,212 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43374, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T17:16:24,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T17:16:24,225 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T17:16:24,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-03T17:16:24,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741920_1096 (size=1097) 2024-12-03T17:16:24,720 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-03T17:16:24,721 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-03T17:16:24,733 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T17:16:24,743 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4e02d2d095200ac1f1a0b05c28cb936a, REOPEN/MOVE}] 2024-12-03T17:16:24,745 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4e02d2d095200ac1f1a0b05c28cb936a, REOPEN/MOVE 2024-12-03T17:16:24,747 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=4e02d2d095200ac1f1a0b05c28cb936a, regionState=CLOSING, regionLocation=27e0f852a164,41167,1733246151593 2024-12-03T17:16:24,755 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4e02d2d095200ac1f1a0b05c28cb936a, REOPEN/MOVE because future has completed 2024-12-03T17:16:24,759 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T17:16:24,760 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593}] 2024-12-03T17:16:24,916 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(122): Close 4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:24,916 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T17:16:24,917 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1722): Closing 4e02d2d095200ac1f1a0b05c28cb936a, disabling compactions & flushes 2024-12-03T17:16:24,917 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:24,917 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:24,917 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. after waiting 0 ms 2024-12-03T17:16:24,917 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:24,946 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T17:16:24,950 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:24,951 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1676): Region close journal for 4e02d2d095200ac1f1a0b05c28cb936a: Waiting for close lock at 1733246184917Running coprocessor pre-close hooks at 1733246184917Disabling compacts and flushes for region at 1733246184917Disabling writes for close at 1733246184917Writing region close event to WAL at 1733246184926 (+9 ms)Running coprocessor post-close hooks at 1733246184950 (+24 ms)Closed at 1733246184950 2024-12-03T17:16:24,951 WARN [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionServer(3299): Not adding moved region record: 4e02d2d095200ac1f1a0b05c28cb936a to self. 2024-12-03T17:16:24,956 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(157): Closed 4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:24,956 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=4e02d2d095200ac1f1a0b05c28cb936a, regionState=CLOSED 2024-12-03T17:16:24,961 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 because future has completed 2024-12-03T17:16:24,971 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-12-03T17:16:24,971 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseRegionProcedure 4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 in 208 msec 2024-12-03T17:16:24,973 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4e02d2d095200ac1f1a0b05c28cb936a, REOPEN/MOVE; state=CLOSED, location=27e0f852a164,41167,1733246151593; forceNewPlan=false, retain=true 2024-12-03T17:16:25,124 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=4e02d2d095200ac1f1a0b05c28cb936a, regionState=OPENING, regionLocation=27e0f852a164,41167,1733246151593 2024-12-03T17:16:25,127 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4e02d2d095200ac1f1a0b05c28cb936a, REOPEN/MOVE because future has completed 2024-12-03T17:16:25,128 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593}] 2024-12-03T17:16:25,286 INFO [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:25,286 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} 2024-12-03T17:16:25,286 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:25,286 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T17:16:25,287 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:25,287 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:25,293 INFO [StoreOpener-4e02d2d095200ac1f1a0b05c28cb936a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:25,294 INFO [StoreOpener-4e02d2d095200ac1f1a0b05c28cb936a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T17:16:25,301 INFO [StoreOpener-4e02d2d095200ac1f1a0b05c28cb936a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4e02d2d095200ac1f1a0b05c28cb936a columnFamilyName A 2024-12-03T17:16:25,305 DEBUG [StoreOpener-4e02d2d095200ac1f1a0b05c28cb936a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:25,306 INFO [StoreOpener-4e02d2d095200ac1f1a0b05c28cb936a-1 {}] regionserver.HStore(327): Store=4e02d2d095200ac1f1a0b05c28cb936a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T17:16:25,307 INFO [StoreOpener-4e02d2d095200ac1f1a0b05c28cb936a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:25,308 INFO [StoreOpener-4e02d2d095200ac1f1a0b05c28cb936a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T17:16:25,309 INFO [StoreOpener-4e02d2d095200ac1f1a0b05c28cb936a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4e02d2d095200ac1f1a0b05c28cb936a columnFamilyName B 2024-12-03T17:16:25,309 DEBUG [StoreOpener-4e02d2d095200ac1f1a0b05c28cb936a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:25,310 INFO [StoreOpener-4e02d2d095200ac1f1a0b05c28cb936a-1 {}] regionserver.HStore(327): Store=4e02d2d095200ac1f1a0b05c28cb936a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T17:16:25,310 INFO [StoreOpener-4e02d2d095200ac1f1a0b05c28cb936a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:25,311 INFO [StoreOpener-4e02d2d095200ac1f1a0b05c28cb936a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T17:16:25,311 INFO [StoreOpener-4e02d2d095200ac1f1a0b05c28cb936a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4e02d2d095200ac1f1a0b05c28cb936a columnFamilyName C 2024-12-03T17:16:25,311 DEBUG [StoreOpener-4e02d2d095200ac1f1a0b05c28cb936a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:25,312 INFO [StoreOpener-4e02d2d095200ac1f1a0b05c28cb936a-1 {}] regionserver.HStore(327): Store=4e02d2d095200ac1f1a0b05c28cb936a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T17:16:25,312 INFO [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:25,312 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:25,313 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:25,315 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:25,317 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:25,317 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:25,318 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T17:16:25,321 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:25,323 INFO [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 4e02d2d095200ac1f1a0b05c28cb936a; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62488104, jitterRate=-0.0688546895980835}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T17:16:25,323 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:25,324 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 4e02d2d095200ac1f1a0b05c28cb936a: Running coprocessor pre-open hook at 1733246185287Writing region info on filesystem at 1733246185287Initializing all the Stores at 1733246185288 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733246185288Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733246185288Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733246185288Cleaning up temporary data from old regions at 1733246185317 (+29 ms)Running coprocessor post-open hooks at 1733246185323 (+6 ms)Region opened successfully at 1733246185324 (+1 ms) 2024-12-03T17:16:25,325 INFO [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., pid=33, masterSystemTime=1733246185281 2024-12-03T17:16:25,341 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=4e02d2d095200ac1f1a0b05c28cb936a, regionState=OPEN, openSeqNum=5, regionLocation=27e0f852a164,41167,1733246151593 2024-12-03T17:16:25,344 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 because future has completed 2024-12-03T17:16:25,346 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:25,346 INFO [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:25,348 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=31 2024-12-03T17:16:25,349 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure 4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 in 217 msec 2024-12-03T17:16:25,354 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-12-03T17:16:25,354 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4e02d2d095200ac1f1a0b05c28cb936a, REOPEN/MOVE in 606 msec 2024-12-03T17:16:25,360 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-03T17:16:25,360 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 624 msec 2024-12-03T17:16:25,387 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 1.1380 sec 2024-12-03T17:16:25,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-03T17:16:25,400 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:16:25,403 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@681c9b14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:25,403 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:16:25,403 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:16:25,405 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:16:25,405 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:16:25,405 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:16:25,406 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74244c81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:25,406 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:16:25,406 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:16:25,406 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:25,408 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43392, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:16:25,409 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c4809f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:25,411 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35d0a90e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:25,411 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:16:25,411 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:16:25,412 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:16:25,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:16:25,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:16:25,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54ed4814, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:25,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:16:25,414 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:16:25,414 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:25,415 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43418, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:16:25,417 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11a9daa2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:25,418 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@771f5e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:25,418 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:16:25,418 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:16:25,420 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:16:25,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:16:25,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:16:25,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b87266f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:25,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:16:25,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:16:25,421 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:25,422 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43432, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:16:25,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f9f9051, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:25,427 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26dcfede, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:25,427 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:16:25,427 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:16:25,428 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:16:25,428 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:16:25,428 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:16:25,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4710d18b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:25,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:16:25,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:16:25,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:25,430 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43438, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:16:25,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38217286, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:25,432 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d21c225, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:25,432 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:16:25,432 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:16:25,433 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:16:25,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:16:25,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:16:25,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d14bba5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:25,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:16:25,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:16:25,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:25,435 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43456, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:16:25,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@377aa18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:25,443 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bbb11d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:25,443 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:16:25,443 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:16:25,444 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:16:25,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:16:25,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:16:25,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e67a7cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:25,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:16:25,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:16:25,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:25,447 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43474, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:16:25,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71ebde08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:25,449 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41d2a6fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:25,449 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:16:25,449 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:16:25,450 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:16:25,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:16:25,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:16:25,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ac1445e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:25,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:16:25,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:16:25,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:25,452 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43504, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:16:25,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53f62ce7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:25,455 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30ade04f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:25,455 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:16:25,455 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:16:25,456 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:16:25,456 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:16:25,456 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:16:25,456 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@159fe5da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:25,457 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:16:25,457 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:16:25,457 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:25,458 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43524, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:16:25,459 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@538759e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:25,460 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62750578, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:25,460 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:16:25,460 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:16:25,461 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:16:25,461 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:16:25,462 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:16:25,462 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dcb8acf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:25,462 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:16:25,462 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:16:25,462 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:25,466 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43544, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:16:25,467 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f33b51f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:25,475 DEBUG [Thread-520 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:16:25,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:16:25,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:16:25,479 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38960, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:16:25,487 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5] 2024-12-03T17:16:25,490 DEBUG [Thread-518 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:16:25,497 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:16:25,497 DEBUG [Thread-529 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:16:25,497 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:16:25,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:16:25,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:16:25,503 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:16:25,504 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38984, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:16:25,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5] 2024-12-03T17:16:25,507 DEBUG [Thread-524 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:16:25,507 DEBUG [Thread-531 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:16:25,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5] 2024-12-03T17:16:25,511 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:16:25,511 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:16:25,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:16:25,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:16:25,513 DEBUG [Thread-533 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:16:25,513 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38996, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:16:25,513 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39010, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:16:25,514 DEBUG [Thread-527 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:16:25,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5] 2024-12-03T17:16:25,517 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T17:16:25,517 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:16:25,518 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:16:25,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees 2024-12-03T17:16:25,519 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39012, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:16:25,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:16:25,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:16:25,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5] 2024-12-03T17:16:25,521 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39016, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:16:25,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-03T17:16:25,522 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T17:16:25,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5] 2024-12-03T17:16:25,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5] 2024-12-03T17:16:25,525 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=34, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T17:16:25,526 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T17:16:25,527 DEBUG [Thread-522 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:16:25,529 DEBUG [Thread-522 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:16:25,529 DEBUG [Thread-522 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:16:25,531 DEBUG [Thread-516 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:16:25,532 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:16:25,532 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:16:25,535 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39046, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:16:25,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:25,537 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4e02d2d095200ac1f1a0b05c28cb936a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T17:16:25,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e02d2d095200ac1f1a0b05c28cb936a, store=A 2024-12-03T17:16:25,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:25,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5] 2024-12-03T17:16:25,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e02d2d095200ac1f1a0b05c28cb936a, store=B 2024-12-03T17:16:25,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:25,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e02d2d095200ac1f1a0b05c28cb936a, store=C 2024-12-03T17:16:25,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:25,539 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39032, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:16:25,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5] 2024-12-03T17:16:25,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-03T17:16:25,678 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733246185622, max timestamp=1733246185668Num uniques 859; 2024-12-03T17:16:25,678 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T17:16:25,678 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733246185622, max timestamp=1733246185666Num uniques 859; 2024-12-03T17:16:25,678 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T17:16:25,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-12-03T17:16:25,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:25,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. as already flushing 2024-12-03T17:16:25,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:25,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:25,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:25,686 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733246185622, max timestamp=1733246185666Num uniques 859; 2024-12-03T17:16:25,686 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T17:16:25,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:25,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412030bff014e774c4c43b46ce7d815875091_4e02d2d095200ac1f1a0b05c28cb936a is 50, key is test_row_0/A:col10/1733246185531/Put/seqid=0 2024-12-03T17:16:25,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741921_1097 (size=17034) 2024-12-03T17:16:25,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:25,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246245755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:25,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:25,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246245761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:25,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:25,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:25,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:25,775 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:25,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:25,775 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:25,775 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:25,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246245762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:25,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:25,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246245765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:25,776 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:25,776 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:25,776 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:25,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:25,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:25,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:25,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:25,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39046 deadline: 1733246245773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:25,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:25,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:25,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:25,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-03T17:16:25,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-12-03T17:16:25,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:25,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. as already flushing 2024-12-03T17:16:25,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:25,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:25,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:25,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:25,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:25,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246245885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:25,895 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:25,895 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:25,895 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:25,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:25,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246245885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:25,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:25,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246245885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:25,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:25,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246245888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:25,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:25,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:25,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:25,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:25,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:25,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:25,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:25,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:25,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:25,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:25,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39046 deadline: 1733246245894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:25,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:25,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:25,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:26,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-12-03T17:16:26,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:26,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. as already flushing 2024-12-03T17:16:26,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:26,027 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:26,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:26,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:26,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:26,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246246105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:26,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:26,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:26,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:26,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:26,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246246105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:26,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:26,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:26,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:26,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:26,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246246107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:26,115 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:26,115 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:26,115 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:26,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:26,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39046 deadline: 1733246246115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:26,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:26,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246246114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:26,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:26,132 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:26,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:26,132 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:26,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:26,132 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:26,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-03T17:16:26,144 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:26,191 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412030bff014e774c4c43b46ce7d815875091_4e02d2d095200ac1f1a0b05c28cb936a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412030bff014e774c4c43b46ce7d815875091_4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:26,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-12-03T17:16:26,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:26,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. as already flushing 2024-12-03T17:16:26,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:26,203 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/2c9b653936e64816a52127851c123cb6, store: [table=TestAcidGuarantees family=A region=4e02d2d095200ac1f1a0b05c28cb936a] 2024-12-03T17:16:26,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:26,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:26,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:26,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/2c9b653936e64816a52127851c123cb6 is 175, key is test_row_0/A:col10/1733246185531/Put/seqid=0 2024-12-03T17:16:26,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741922_1098 (size=48139) 2024-12-03T17:16:26,298 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/2c9b653936e64816a52127851c123cb6 2024-12-03T17:16:26,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-12-03T17:16:26,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:26,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. as already flushing 2024-12-03T17:16:26,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:26,376 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:26,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:26,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:26,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/e6f8efa8a5e242a69df9959922fb65d1 is 50, key is test_row_0/B:col10/1733246185531/Put/seqid=0 2024-12-03T17:16:26,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741923_1099 (size=12001) 2024-12-03T17:16:26,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246246424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:26,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:26,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:26,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:26,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:26,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246246427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:26,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:26,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:26,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:26,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246246431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:26,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:26,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:26,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:26,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:26,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246246444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:26,452 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:26,452 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:26,452 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:26,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:26,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39046 deadline: 1733246246449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:26,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:26,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:26,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:26,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-12-03T17:16:26,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:26,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. as already flushing 2024-12-03T17:16:26,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:26,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:26,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:26,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:26,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-03T17:16:26,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-12-03T17:16:26,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:26,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. as already flushing 2024-12-03T17:16:26,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:26,711 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:26,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:26,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:26,826 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/e6f8efa8a5e242a69df9959922fb65d1 2024-12-03T17:16:26,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-12-03T17:16:26,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:26,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. as already flushing 2024-12-03T17:16:26,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:26,872 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:26,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:26,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:26,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/8911ef322c5f46bc83d81f738b59fcdc is 50, key is test_row_0/C:col10/1733246185531/Put/seqid=0 2024-12-03T17:16:26,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741924_1100 (size=12001) 2024-12-03T17:16:26,935 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/8911ef322c5f46bc83d81f738b59fcdc 2024-12-03T17:16:26,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:26,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246246934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:26,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:26,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:26,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:26,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:26,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246246951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:26,958 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:26,958 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:26,958 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:26,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:26,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246246951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:26,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:26,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:26,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:26,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:26,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39046 deadline: 1733246246964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:26,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:26,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:26,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:26,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:26,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246246964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:26,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:26,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:26,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:26,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/2c9b653936e64816a52127851c123cb6 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/2c9b653936e64816a52127851c123cb6 2024-12-03T17:16:27,023 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/2c9b653936e64816a52127851c123cb6, entries=250, sequenceid=14, filesize=47.0 K 2024-12-03T17:16:27,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/e6f8efa8a5e242a69df9959922fb65d1 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/e6f8efa8a5e242a69df9959922fb65d1 2024-12-03T17:16:27,040 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-12-03T17:16:27,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:27,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. as already flushing 2024-12-03T17:16:27,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:27,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:27,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:27,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:27,043 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/e6f8efa8a5e242a69df9959922fb65d1, entries=150, sequenceid=14, filesize=11.7 K 2024-12-03T17:16:27,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/8911ef322c5f46bc83d81f738b59fcdc as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/8911ef322c5f46bc83d81f738b59fcdc 2024-12-03T17:16:27,065 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/8911ef322c5f46bc83d81f738b59fcdc, entries=150, sequenceid=14, filesize=11.7 K 2024-12-03T17:16:27,066 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 4e02d2d095200ac1f1a0b05c28cb936a in 1530ms, sequenceid=14, compaction requested=false 2024-12-03T17:16:27,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4e02d2d095200ac1f1a0b05c28cb936a: 2024-12-03T17:16:27,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-12-03T17:16:27,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:27,196 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2902): Flushing 4e02d2d095200ac1f1a0b05c28cb936a 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-03T17:16:27,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e02d2d095200ac1f1a0b05c28cb936a, store=A 2024-12-03T17:16:27,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:27,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e02d2d095200ac1f1a0b05c28cb936a, store=B 2024-12-03T17:16:27,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:27,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e02d2d095200ac1f1a0b05c28cb936a, store=C 2024-12-03T17:16:27,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:27,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203e7520f740a034a10b100a57f4948670b_4e02d2d095200ac1f1a0b05c28cb936a is 50, key is test_row_0/A:col10/1733246185760/Put/seqid=0 2024-12-03T17:16:27,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741925_1101 (size=12154) 2024-12-03T17:16:27,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:27,293 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203e7520f740a034a10b100a57f4948670b_4e02d2d095200ac1f1a0b05c28cb936a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203e7520f740a034a10b100a57f4948670b_4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:27,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/bae8301b231245ea8c7173db21977730, store: [table=TestAcidGuarantees family=A region=4e02d2d095200ac1f1a0b05c28cb936a] 2024-12-03T17:16:27,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/bae8301b231245ea8c7173db21977730 is 175, key is test_row_0/A:col10/1733246185760/Put/seqid=0 2024-12-03T17:16:27,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741926_1102 (size=30955) 2024-12-03T17:16:27,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-03T17:16:27,763 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/bae8301b231245ea8c7173db21977730 2024-12-03T17:16:27,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/2c3761f2c37443249a553c32a0b9c895 is 50, key is test_row_0/B:col10/1733246185760/Put/seqid=0 2024-12-03T17:16:27,872 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T17:16:27,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741927_1103 (size=12001) 2024-12-03T17:16:27,880 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/2c3761f2c37443249a553c32a0b9c895 2024-12-03T17:16:27,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/9c246221e8f84e909b6ac1be23c0a4ba is 50, key is test_row_0/C:col10/1733246185760/Put/seqid=0 2024-12-03T17:16:27,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. as already flushing 2024-12-03T17:16:27,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:27,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741928_1104 (size=12001) 2024-12-03T17:16:27,980 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/9c246221e8f84e909b6ac1be23c0a4ba 2024-12-03T17:16:27,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/bae8301b231245ea8c7173db21977730 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/bae8301b231245ea8c7173db21977730 2024-12-03T17:16:28,003 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/bae8301b231245ea8c7173db21977730, entries=150, sequenceid=50, filesize=30.2 K 2024-12-03T17:16:28,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/2c3761f2c37443249a553c32a0b9c895 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/2c3761f2c37443249a553c32a0b9c895 2024-12-03T17:16:28,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:28,012 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/2c3761f2c37443249a553c32a0b9c895, entries=150, sequenceid=50, filesize=11.7 K 2024-12-03T17:16:28,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/9c246221e8f84e909b6ac1be23c0a4ba as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/9c246221e8f84e909b6ac1be23c0a4ba 2024-12-03T17:16:28,024 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/9c246221e8f84e909b6ac1be23c0a4ba, entries=150, sequenceid=50, filesize=11.7 K 2024-12-03T17:16:28,031 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=40.25 KB/41220 for 4e02d2d095200ac1f1a0b05c28cb936a in 835ms, sequenceid=50, compaction requested=false 2024-12-03T17:16:28,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2603): Flush status journal for 4e02d2d095200ac1f1a0b05c28cb936a: 2024-12-03T17:16:28,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:28,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=35 2024-12-03T17:16:28,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:28,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4169): Remote procedure done, pid=35 2024-12-03T17:16:28,033 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4e02d2d095200ac1f1a0b05c28cb936a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T17:16:28,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e02d2d095200ac1f1a0b05c28cb936a, store=A 2024-12-03T17:16:28,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:28,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e02d2d095200ac1f1a0b05c28cb936a, store=B 2024-12-03T17:16:28,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:28,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e02d2d095200ac1f1a0b05c28cb936a, store=C 2024-12-03T17:16:28,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:28,043 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-12-03T17:16:28,043 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5130 sec 2024-12-03T17:16:28,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:28,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:28,048 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees in 2.5290 sec 2024-12-03T17:16:28,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:28,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:28,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412037cef23a585444199bc282af113c7738d_4e02d2d095200ac1f1a0b05c28cb936a is 50, key is test_row_0/A:col10/1733246188026/Put/seqid=0 2024-12-03T17:16:28,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:28,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:28,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:28,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:28,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:28,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:28,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:28,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:28,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:28,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:28,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:28,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:28,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:28,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:28,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:28,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:28,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:28,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:28,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741929_1105 (size=19474) 2024-12-03T17:16:28,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:28,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:28,120 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:28,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:28,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:28,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:28,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:28,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:28,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:28,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:28,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:28,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:28,142 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412037cef23a585444199bc282af113c7738d_4e02d2d095200ac1f1a0b05c28cb936a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412037cef23a585444199bc282af113c7738d_4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:28,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:28,146 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/8739b0eebdc64c60baf13f987ce3dd11, store: [table=TestAcidGuarantees family=A region=4e02d2d095200ac1f1a0b05c28cb936a] 2024-12-03T17:16:28,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/8739b0eebdc64c60baf13f987ce3dd11 is 175, key is test_row_0/A:col10/1733246188026/Put/seqid=0 2024-12-03T17:16:28,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:28,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:28,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:28,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:28,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:28,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:28,163 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1733246188037, max timestamp=1733246188160Num uniques 859; 2024-12-03T17:16:28,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:28,163 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-12-03T17:16:28,166 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1733246188037, max timestamp=1733246188160Num uniques 859; 2024-12-03T17:16:28,166 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-12-03T17:16:28,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:28,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:28,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:28,170 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1733246188037, max timestamp=1733246188160Num uniques 859; 2024-12-03T17:16:28,170 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-12-03T17:16:28,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:28,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:28,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:28,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:28,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:28,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:28,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246248181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:28,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:28,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246248183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:28,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:28,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246248186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:28,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:28,203 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:28,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:28,203 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:28,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:28,203 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:28,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:28,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:28,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:28,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741930_1106 (size=56733) 2024-12-03T17:16:28,208 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=60, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/8739b0eebdc64c60baf13f987ce3dd11 2024-12-03T17:16:28,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:28,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39046 deadline: 1733246248203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:28,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:28,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:28,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:28,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:28,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246248197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:28,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:28,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:28,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:28,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/ffb075ec355144cf904297a5949bd461 is 50, key is test_row_0/B:col10/1733246188026/Put/seqid=0 2024-12-03T17:16:28,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741931_1107 (size=12001) 2024-12-03T17:16:28,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:28,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246248315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:28,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:28,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:28,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246248315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:28,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:28,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:28,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:28,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246248315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:28,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:28,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:28,330 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:28,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:28,330 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:28,330 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:28,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:28,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246248335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:28,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:28,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:28,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:28,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:28,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39046 deadline: 1733246248347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:28,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:28,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:28,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:28,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:28,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246248533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:28,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:28,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:28,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:28,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:28,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246248539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:28,548 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:28,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:28,548 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:28,548 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:28,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246248534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:28,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:28,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:28,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:28,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:28,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246248543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:28,550 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:28,550 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:28,550 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:28,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:28,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39046 deadline: 1733246248555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:28,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:28,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:28,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:28,699 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/ffb075ec355144cf904297a5949bd461 2024-12-03T17:16:28,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/f0de18dc3a1b45d299aa119c531a1127 is 50, key is test_row_0/C:col10/1733246188026/Put/seqid=0 2024-12-03T17:16:28,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741932_1108 (size=12001) 2024-12-03T17:16:28,832 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/f0de18dc3a1b45d299aa119c531a1127 2024-12-03T17:16:28,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:28,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246248854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:28,861 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:28,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:28,861 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:28,862 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:28,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246248855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:28,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:28,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:28,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:28,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:28,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:28,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246248855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:28,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:28,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:28,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:28,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246248855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:28,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:28,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:28,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:28,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:28,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39046 deadline: 1733246248884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:28,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:28,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:28,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:28,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/8739b0eebdc64c60baf13f987ce3dd11 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/8739b0eebdc64c60baf13f987ce3dd11 2024-12-03T17:16:28,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/8739b0eebdc64c60baf13f987ce3dd11, entries=300, sequenceid=60, filesize=55.4 K 2024-12-03T17:16:29,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/ffb075ec355144cf904297a5949bd461 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/ffb075ec355144cf904297a5949bd461 2024-12-03T17:16:29,057 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/ffb075ec355144cf904297a5949bd461, entries=150, sequenceid=60, filesize=11.7 K 2024-12-03T17:16:29,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/f0de18dc3a1b45d299aa119c531a1127 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/f0de18dc3a1b45d299aa119c531a1127 2024-12-03T17:16:29,105 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/f0de18dc3a1b45d299aa119c531a1127, entries=150, sequenceid=60, filesize=11.7 K 2024-12-03T17:16:29,107 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=154.31 KB/158010 for 4e02d2d095200ac1f1a0b05c28cb936a in 1074ms, sequenceid=60, compaction requested=true 2024-12-03T17:16:29,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4e02d2d095200ac1f1a0b05c28cb936a: 2024-12-03T17:16:29,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e02d2d095200ac1f1a0b05c28cb936a:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T17:16:29,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:16:29,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e02d2d095200ac1f1a0b05c28cb936a:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T17:16:29,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T17:16:29,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e02d2d095200ac1f1a0b05c28cb936a:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T17:16:29,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-03T17:16:29,108 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:16:29,111 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 135827 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:16:29,111 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): 4e02d2d095200ac1f1a0b05c28cb936a/A is initiating minor compaction (all files) 2024-12-03T17:16:29,111 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e02d2d095200ac1f1a0b05c28cb936a/A in TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:29,111 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/2c9b653936e64816a52127851c123cb6, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/bae8301b231245ea8c7173db21977730, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/8739b0eebdc64c60baf13f987ce3dd11] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp, totalSize=132.6 K 2024-12-03T17:16:29,111 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:29,112 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. files: [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/2c9b653936e64816a52127851c123cb6, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/bae8301b231245ea8c7173db21977730, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/8739b0eebdc64c60baf13f987ce3dd11] 2024-12-03T17:16:29,113 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c9b653936e64816a52127851c123cb6, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733246185490 2024-12-03T17:16:29,114 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:16:29,115 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting bae8301b231245ea8c7173db21977730, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733246185757 2024-12-03T17:16:29,116 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 8739b0eebdc64c60baf13f987ce3dd11, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733246187976 2024-12-03T17:16:29,126 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:16:29,127 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): 4e02d2d095200ac1f1a0b05c28cb936a/C is initiating minor compaction (all files) 2024-12-03T17:16:29,127 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e02d2d095200ac1f1a0b05c28cb936a/C in TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:29,127 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/8911ef322c5f46bc83d81f738b59fcdc, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/9c246221e8f84e909b6ac1be23c0a4ba, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/f0de18dc3a1b45d299aa119c531a1127] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp, totalSize=35.2 K 2024-12-03T17:16:29,139 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8911ef322c5f46bc83d81f738b59fcdc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733246185527 2024-12-03T17:16:29,143 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c246221e8f84e909b6ac1be23c0a4ba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733246185757 2024-12-03T17:16:29,145 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0de18dc3a1b45d299aa119c531a1127, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733246187988 2024-12-03T17:16:29,150 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4e02d2d095200ac1f1a0b05c28cb936a] 2024-12-03T17:16:29,174 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120394b3b2b8c0cd4310a707e9c1e30300ee_4e02d2d095200ac1f1a0b05c28cb936a store=[table=TestAcidGuarantees family=A region=4e02d2d095200ac1f1a0b05c28cb936a] 2024-12-03T17:16:29,179 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120394b3b2b8c0cd4310a707e9c1e30300ee_4e02d2d095200ac1f1a0b05c28cb936a, store=[table=TestAcidGuarantees family=A region=4e02d2d095200ac1f1a0b05c28cb936a] 2024-12-03T17:16:29,179 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120394b3b2b8c0cd4310a707e9c1e30300ee_4e02d2d095200ac1f1a0b05c28cb936a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4e02d2d095200ac1f1a0b05c28cb936a] 2024-12-03T17:16:29,270 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e02d2d095200ac1f1a0b05c28cb936a#C#compaction#91 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:16:29,270 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/e7b73b54d2924a0fa777fd318b512704 is 50, key is test_row_0/C:col10/1733246188026/Put/seqid=0 2024-12-03T17:16:29,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741933_1109 (size=4469) 2024-12-03T17:16:29,287 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e02d2d095200ac1f1a0b05c28cb936a#A#compaction#90 average throughput is 0.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:16:29,288 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/b11c602c8af84ebb8b7bef56c24f0975 is 175, key is test_row_0/A:col10/1733246188026/Put/seqid=0 2024-12-03T17:16:29,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741934_1110 (size=12104) 2024-12-03T17:16:29,348 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/e7b73b54d2924a0fa777fd318b512704 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/e7b73b54d2924a0fa777fd318b512704 2024-12-03T17:16:29,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741935_1111 (size=31058) 2024-12-03T17:16:29,376 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4e02d2d095200ac1f1a0b05c28cb936a/C of 4e02d2d095200ac1f1a0b05c28cb936a into e7b73b54d2924a0fa777fd318b512704(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:16:29,376 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e02d2d095200ac1f1a0b05c28cb936a: 2024-12-03T17:16:29,376 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., storeName=4e02d2d095200ac1f1a0b05c28cb936a/C, priority=13, startTime=1733246189108; duration=0sec 2024-12-03T17:16:29,376 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:16:29,376 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e02d2d095200ac1f1a0b05c28cb936a:C 2024-12-03T17:16:29,377 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:16:29,382 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/b11c602c8af84ebb8b7bef56c24f0975 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/b11c602c8af84ebb8b7bef56c24f0975 2024-12-03T17:16:29,390 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4e02d2d095200ac1f1a0b05c28cb936a 3/3 column families, dataSize=161.02 KB heapSize=399.05 KB 2024-12-03T17:16:29,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e02d2d095200ac1f1a0b05c28cb936a, store=A 2024-12-03T17:16:29,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:29,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e02d2d095200ac1f1a0b05c28cb936a, store=B 2024-12-03T17:16:29,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:29,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e02d2d095200ac1f1a0b05c28cb936a, store=C 2024-12-03T17:16:29,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:29,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:29,392 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:16:29,392 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): 4e02d2d095200ac1f1a0b05c28cb936a/B is initiating minor compaction (all files) 2024-12-03T17:16:29,392 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e02d2d095200ac1f1a0b05c28cb936a/B in TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:29,392 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/e6f8efa8a5e242a69df9959922fb65d1, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/2c3761f2c37443249a553c32a0b9c895, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/ffb075ec355144cf904297a5949bd461] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp, totalSize=35.2 K 2024-12-03T17:16:29,394 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6f8efa8a5e242a69df9959922fb65d1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733246185527 2024-12-03T17:16:29,394 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c3761f2c37443249a553c32a0b9c895, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733246185757 2024-12-03T17:16:29,395 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffb075ec355144cf904297a5949bd461, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733246187988 2024-12-03T17:16:29,399 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4e02d2d095200ac1f1a0b05c28cb936a/A of 4e02d2d095200ac1f1a0b05c28cb936a into b11c602c8af84ebb8b7bef56c24f0975(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:16:29,399 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e02d2d095200ac1f1a0b05c28cb936a: 2024-12-03T17:16:29,399 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., storeName=4e02d2d095200ac1f1a0b05c28cb936a/A, priority=13, startTime=1733246189107; duration=0sec 2024-12-03T17:16:29,399 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:29,399 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e02d2d095200ac1f1a0b05c28cb936a:A 2024-12-03T17:16:29,425 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e02d2d095200ac1f1a0b05c28cb936a#B#compaction#92 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:16:29,426 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/8e3613a780b24a62ae00311c53459bc4 is 50, key is test_row_0/B:col10/1733246188026/Put/seqid=0 2024-12-03T17:16:29,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:29,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246249402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:29,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:29,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39046 deadline: 1733246249410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:29,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:29,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:29,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:29,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:29,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:29,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:29,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:29,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246249425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:29,443 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:29,443 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:29,443 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:29,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:29,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246249429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:29,444 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203a08de2a084434707928e985d0f3aa704_4e02d2d095200ac1f1a0b05c28cb936a is 50, key is test_row_0/A:col10/1733246189384/Put/seqid=0 2024-12-03T17:16:29,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:29,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:29,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:29,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:29,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246249430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:29,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:29,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:29,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:29,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741936_1112 (size=12104) 2024-12-03T17:16:29,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:29,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246249534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:29,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:29,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:29,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:29,550 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/8e3613a780b24a62ae00311c53459bc4 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/8e3613a780b24a62ae00311c53459bc4 2024-12-03T17:16:29,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741937_1113 (size=14594) 2024-12-03T17:16:29,565 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4e02d2d095200ac1f1a0b05c28cb936a/B of 4e02d2d095200ac1f1a0b05c28cb936a into 8e3613a780b24a62ae00311c53459bc4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:16:29,565 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e02d2d095200ac1f1a0b05c28cb936a: 2024-12-03T17:16:29,565 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., storeName=4e02d2d095200ac1f1a0b05c28cb936a/B, priority=13, startTime=1733246189107; duration=0sec 2024-12-03T17:16:29,566 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:29,566 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e02d2d095200ac1f1a0b05c28cb936a:B 2024-12-03T17:16:29,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:29,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246249554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:29,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:29,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246249554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:29,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:29,568 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:29,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246249554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:29,568 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:29,568 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:29,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:29,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:29,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:29,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:29,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:29,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:29,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-03T17:16:29,703 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:16:29,704 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T17:16:29,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-12-03T17:16:29,712 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T17:16:29,713 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T17:16:29,713 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T17:16:29,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-03T17:16:29,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:29,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246249754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:29,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:29,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:29,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:29,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:29,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246249774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:29,800 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:29,800 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:29,801 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:29,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:29,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246249775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:29,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:29,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:29,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:29,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:29,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246249776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:29,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:29,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:29,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:29,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-03T17:16:29,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-03T17:16:29,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:29,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. as already flushing 2024-12-03T17:16:29,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:29,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:29,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:29,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:29,970 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:29,997 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203a08de2a084434707928e985d0f3aa704_4e02d2d095200ac1f1a0b05c28cb936a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203a08de2a084434707928e985d0f3aa704_4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:30,003 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/798c3d2a8f0c4c36a06ca3863336ac87, store: [table=TestAcidGuarantees family=A region=4e02d2d095200ac1f1a0b05c28cb936a] 2024-12-03T17:16:30,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/798c3d2a8f0c4c36a06ca3863336ac87 is 175, key is test_row_0/A:col10/1733246189384/Put/seqid=0 2024-12-03T17:16:30,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-03T17:16:30,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741938_1114 (size=39549) 2024-12-03T17:16:30,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-03T17:16:30,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:30,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. as already flushing 2024-12-03T17:16:30,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:30,063 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:30,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:30,064 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=88, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/798c3d2a8f0c4c36a06ca3863336ac87 2024-12-03T17:16:30,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:30,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246250064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:30,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:30,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:30,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:30,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:30,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:30,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246250107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:30,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:30,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246250114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:30,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:30,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:30,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:30,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:30,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:30,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:30,133 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/61793b763dba4d42bfb66476f935428c is 50, key is test_row_0/B:col10/1733246189384/Put/seqid=0 2024-12-03T17:16:30,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:30,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246250122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:30,143 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:30,143 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:30,143 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:30,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741939_1115 (size=12001) 2024-12-03T17:16:30,210 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/61793b763dba4d42bfb66476f935428c 2024-12-03T17:16:30,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-03T17:16:30,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:30,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. as already flushing 2024-12-03T17:16:30,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:30,243 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:30,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:30,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:30,245 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/615be05b4f5345ec876cfc3805a0de45 is 50, key is test_row_0/C:col10/1733246189384/Put/seqid=0 2024-12-03T17:16:30,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741940_1116 (size=12001) 2024-12-03T17:16:30,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-03T17:16:30,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-03T17:16:30,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:30,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. as already flushing 2024-12-03T17:16:30,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:30,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:30,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:30,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:30,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:30,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39046 deadline: 1733246250447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:30,457 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 27e0f852a164,41167,1733246151593 for 'test_row_0' in 4e02d2d095200ac1f1a0b05c28cb936a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2254 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T17:16:30,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:30,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:30,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:30,562 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-03T17:16:30,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:30,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. as already flushing 2024-12-03T17:16:30,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:30,562 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:30,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:30,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:30,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:30,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246250586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:30,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:30,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:30,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:30,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:30,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246250624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:30,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:30,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:30,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:30,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:30,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246250634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:30,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:30,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:30,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:30,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:30,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246250653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:30,655 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:30,655 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:30,655 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:30,682 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/615be05b4f5345ec876cfc3805a0de45 2024-12-03T17:16:30,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/798c3d2a8f0c4c36a06ca3863336ac87 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/798c3d2a8f0c4c36a06ca3863336ac87 2024-12-03T17:16:30,704 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/798c3d2a8f0c4c36a06ca3863336ac87, entries=200, sequenceid=88, filesize=38.6 K 2024-12-03T17:16:30,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/61793b763dba4d42bfb66476f935428c as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/61793b763dba4d42bfb66476f935428c 2024-12-03T17:16:30,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-03T17:16:30,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:30,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. as already flushing 2024-12-03T17:16:30,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:30,718 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:30,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:30,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:30,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/61793b763dba4d42bfb66476f935428c, entries=150, sequenceid=88, filesize=11.7 K 2024-12-03T17:16:30,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/615be05b4f5345ec876cfc3805a0de45 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/615be05b4f5345ec876cfc3805a0de45 2024-12-03T17:16:30,737 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/615be05b4f5345ec876cfc3805a0de45, entries=150, sequenceid=88, filesize=11.7 K 2024-12-03T17:16:30,740 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~399 KB/408576, currentSize=40.25 KB/41220 for 4e02d2d095200ac1f1a0b05c28cb936a in 1349ms, sequenceid=88, compaction requested=false 2024-12-03T17:16:30,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4e02d2d095200ac1f1a0b05c28cb936a: 2024-12-03T17:16:30,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-03T17:16:30,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-03T17:16:30,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:30,873 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing 4e02d2d095200ac1f1a0b05c28cb936a 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-03T17:16:30,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e02d2d095200ac1f1a0b05c28cb936a, store=A 2024-12-03T17:16:30,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:30,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e02d2d095200ac1f1a0b05c28cb936a, store=B 2024-12-03T17:16:30,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:30,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e02d2d095200ac1f1a0b05c28cb936a, store=C 2024-12-03T17:16:30,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:30,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203e13194aa7b164ebf87655b899fda471a_4e02d2d095200ac1f1a0b05c28cb936a is 50, key is test_row_0/A:col10/1733246189399/Put/seqid=0 2024-12-03T17:16:30,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741941_1117 (size=12154) 2024-12-03T17:16:31,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:31,375 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203e13194aa7b164ebf87655b899fda471a_4e02d2d095200ac1f1a0b05c28cb936a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203e13194aa7b164ebf87655b899fda471a_4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:31,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/019d440542c54f45b3d09dac09a5c53a, store: [table=TestAcidGuarantees family=A region=4e02d2d095200ac1f1a0b05c28cb936a] 2024-12-03T17:16:31,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/019d440542c54f45b3d09dac09a5c53a is 175, key is test_row_0/A:col10/1733246189399/Put/seqid=0 2024-12-03T17:16:31,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741942_1118 (size=30955) 2024-12-03T17:16:31,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:31,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. as already flushing 2024-12-03T17:16:31,725 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733246191690, max timestamp=1733246191716Num uniques 859; 2024-12-03T17:16:31,725 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T17:16:31,728 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733246191690, max timestamp=1733246191716Num uniques 859; 2024-12-03T17:16:31,729 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T17:16:31,730 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733246191690, max timestamp=1733246191718Num uniques 859; 2024-12-03T17:16:31,730 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T17:16:31,833 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=99, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/019d440542c54f45b3d09dac09a5c53a 2024-12-03T17:16:31,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:31,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246251839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:31,859 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=368, cellSize=16848, totalHeapSize=38512, min timestamp=1733246191776, max timestamp=1733246191848Num uniques 1018; 2024-12-03T17:16:31,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:31,859 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=16848, suffix heap size=133840,new segment heap size=38512 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=368 2024-12-03T17:16:31,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:31,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:31,865 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733246191795, max timestamp=1733246191848Num uniques 1018; 2024-12-03T17:16:31,865 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-12-03T17:16:31,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733246191795, max timestamp=1733246191848Num uniques 1018; 2024-12-03T17:16:31,866 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-12-03T17:16:31,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-03T17:16:31,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/dca990e2a0224cff9b6f43fd5d0e4ca8 is 50, key is test_row_0/B:col10/1733246189399/Put/seqid=0 2024-12-03T17:16:31,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741943_1119 (size=12001) 2024-12-03T17:16:31,924 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/dca990e2a0224cff9b6f43fd5d0e4ca8 2024-12-03T17:16:31,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/0930543cebeb41f2af31f6b43422d954 is 50, key is test_row_0/C:col10/1733246189399/Put/seqid=0 2024-12-03T17:16:31,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741944_1120 (size=12001) 2024-12-03T17:16:31,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:31,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246251974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:32,000 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:32,000 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:32,000 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:32,013 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/0930543cebeb41f2af31f6b43422d954 2024-12-03T17:16:32,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:32,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246251985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:32,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:32,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246251999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:32,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:32,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:32,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:32,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:32,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:32,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:32,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:32,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246251993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:32,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:32,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:32,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:32,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/019d440542c54f45b3d09dac09a5c53a as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/019d440542c54f45b3d09dac09a5c53a 2024-12-03T17:16:32,046 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/019d440542c54f45b3d09dac09a5c53a, entries=150, sequenceid=99, filesize=30.2 K 2024-12-03T17:16:32,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/dca990e2a0224cff9b6f43fd5d0e4ca8 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/dca990e2a0224cff9b6f43fd5d0e4ca8 2024-12-03T17:16:32,057 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/dca990e2a0224cff9b6f43fd5d0e4ca8, entries=150, sequenceid=99, filesize=11.7 K 2024-12-03T17:16:32,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/0930543cebeb41f2af31f6b43422d954 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/0930543cebeb41f2af31f6b43422d954 2024-12-03T17:16:32,079 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/0930543cebeb41f2af31f6b43422d954, entries=150, sequenceid=99, filesize=11.7 K 2024-12-03T17:16:32,091 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=145.36 KB/148850 for 4e02d2d095200ac1f1a0b05c28cb936a in 1218ms, sequenceid=99, compaction requested=true 2024-12-03T17:16:32,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for 4e02d2d095200ac1f1a0b05c28cb936a: 2024-12-03T17:16:32,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:32,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-12-03T17:16:32,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-12-03T17:16:32,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:32,118 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4e02d2d095200ac1f1a0b05c28cb936a 3/3 column families, dataSize=152.07 KB heapSize=424.77 KB 2024-12-03T17:16:32,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e02d2d095200ac1f1a0b05c28cb936a, store=A 2024-12-03T17:16:32,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:32,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e02d2d095200ac1f1a0b05c28cb936a, store=B 2024-12-03T17:16:32,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:32,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e02d2d095200ac1f1a0b05c28cb936a, store=C 2024-12-03T17:16:32,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:32,127 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-12-03T17:16:32,127 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3890 sec 2024-12-03T17:16:32,159 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 2.4340 sec 2024-12-03T17:16:32,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:32,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246252156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:32,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:32,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246252158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:32,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:32,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:32,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:32,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:32,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:32,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:32,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412038a136ee16dc04e08a7318781802a6fa8_4e02d2d095200ac1f1a0b05c28cb936a is 50, key is test_row_0/A:col10/1733246191916/Put/seqid=0 2024-12-03T17:16:32,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:32,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246252168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:32,185 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:32,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:32,186 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:32,186 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:32,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246252171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:32,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:32,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:32,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:32,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741945_1121 (size=12304) 2024-12-03T17:16:32,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:32,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246252286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:32,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:32,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246252287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:32,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:32,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:32,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:32,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:32,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246252293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:32,305 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:32,305 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:32,305 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:32,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:32,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:32,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:32,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:32,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246252302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:32,310 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:32,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:32,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:32,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:32,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39046 deadline: 1733246252484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:32,499 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 27e0f852a164,41167,1733246151593 for 'test_row_0' in 4e02d2d095200ac1f1a0b05c28cb936a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4296 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T17:16:32,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:32,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:32,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:32,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:32,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246252504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:32,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:32,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:32,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:32,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:32,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246252514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:32,517 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:32,517 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:32,517 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:32,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:32,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246252515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:32,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:32,524 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:32,524 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:32,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246252515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:32,524 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:32,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:32,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:32,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:32,622 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:32,663 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412038a136ee16dc04e08a7318781802a6fa8_4e02d2d095200ac1f1a0b05c28cb936a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412038a136ee16dc04e08a7318781802a6fa8_4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:32,674 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/29b6adb605054138af14279dc3c65489, store: [table=TestAcidGuarantees family=A region=4e02d2d095200ac1f1a0b05c28cb936a] 2024-12-03T17:16:32,675 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/29b6adb605054138af14279dc3c65489 is 175, key is test_row_0/A:col10/1733246191916/Put/seqid=0 2024-12-03T17:16:32,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741946_1122 (size=31105) 2024-12-03T17:16:32,735 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=153, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/29b6adb605054138af14279dc3c65489 2024-12-03T17:16:32,804 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/233f9d05676e4be3b928c2f4d0dfa852 is 50, key is test_row_0/B:col10/1733246191916/Put/seqid=0 2024-12-03T17:16:32,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:32,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246252824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:32,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:32,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:32,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:32,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:32,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246252824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:32,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:32,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246252839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:32,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:32,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:32,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246252843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:32,846 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:32,847 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:32,847 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:32,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:32,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:32,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741947_1123 (size=12151) 2024-12-03T17:16:32,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:32,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:32,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:33,259 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/233f9d05676e4be3b928c2f4d0dfa852 2024-12-03T17:16:33,310 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/ab4c98384c744c17842ee49a685b492f is 50, key is test_row_0/C:col10/1733246191916/Put/seqid=0 2024-12-03T17:16:33,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:33,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246253345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:33,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:33,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246253355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:33,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:33,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246253355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:33,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:33,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:33,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:33,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:33,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:33,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:33,360 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:33,360 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:33,360 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:33,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741948_1124 (size=12151) 2024-12-03T17:16:33,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:33,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246253364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:33,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:33,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:33,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:33,765 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/ab4c98384c744c17842ee49a685b492f 2024-12-03T17:16:33,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/29b6adb605054138af14279dc3c65489 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/29b6adb605054138af14279dc3c65489 2024-12-03T17:16:33,831 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/29b6adb605054138af14279dc3c65489, entries=150, sequenceid=153, filesize=30.4 K 2024-12-03T17:16:33,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/233f9d05676e4be3b928c2f4d0dfa852 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/233f9d05676e4be3b928c2f4d0dfa852 2024-12-03T17:16:33,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,843 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/233f9d05676e4be3b928c2f4d0dfa852, entries=150, sequenceid=153, filesize=11.9 K 2024-12-03T17:16:33,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/ab4c98384c744c17842ee49a685b492f as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/ab4c98384c744c17842ee49a685b492f 2024-12-03T17:16:33,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,863 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/ab4c98384c744c17842ee49a685b492f, entries=150, sequenceid=153, filesize=11.9 K 2024-12-03T17:16:33,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,864 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~152.07 KB/155720, heapSize ~424.72 KB/434912, currentSize=40.25 KB/41220 for 4e02d2d095200ac1f1a0b05c28cb936a in 1747ms, sequenceid=153, compaction requested=true 2024-12-03T17:16:33,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4e02d2d095200ac1f1a0b05c28cb936a: 2024-12-03T17:16:33,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,865 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T17:16:33,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e02d2d095200ac1f1a0b05c28cb936a:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T17:16:33,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:33,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,865 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T17:16:33,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e02d2d095200ac1f1a0b05c28cb936a:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T17:16:33,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:33,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e02d2d095200ac1f1a0b05c28cb936a:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T17:16:33,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:16:33,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,869 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T17:16:33,869 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): 4e02d2d095200ac1f1a0b05c28cb936a/B is initiating minor compaction (all files) 2024-12-03T17:16:33,869 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e02d2d095200ac1f1a0b05c28cb936a/B in TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:33,869 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/8e3613a780b24a62ae00311c53459bc4, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/61793b763dba4d42bfb66476f935428c, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/dca990e2a0224cff9b6f43fd5d0e4ca8, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/233f9d05676e4be3b928c2f4d0dfa852] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp, totalSize=47.1 K 2024-12-03T17:16:33,870 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e3613a780b24a62ae00311c53459bc4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733246187988 2024-12-03T17:16:33,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,872 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132667 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T17:16:33,873 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): 4e02d2d095200ac1f1a0b05c28cb936a/A is initiating minor compaction (all files) 2024-12-03T17:16:33,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,873 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e02d2d095200ac1f1a0b05c28cb936a/A in TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:33,873 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/b11c602c8af84ebb8b7bef56c24f0975, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/798c3d2a8f0c4c36a06ca3863336ac87, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/019d440542c54f45b3d09dac09a5c53a, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/29b6adb605054138af14279dc3c65489] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp, totalSize=129.6 K 2024-12-03T17:16:33,873 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:33,873 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. files: [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/b11c602c8af84ebb8b7bef56c24f0975, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/798c3d2a8f0c4c36a06ca3863336ac87, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/019d440542c54f45b3d09dac09a5c53a, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/29b6adb605054138af14279dc3c65489] 2024-12-03T17:16:33,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-03T17:16:33,874 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 61793b763dba4d42bfb66476f935428c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1733246188142 2024-12-03T17:16:33,875 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:16:33,875 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T17:16:33,875 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting dca990e2a0224cff9b6f43fd5d0e4ca8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733246189399 2024-12-03T17:16:33,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,876 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting b11c602c8af84ebb8b7bef56c24f0975, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733246187988 2024-12-03T17:16:33,876 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 233f9d05676e4be3b928c2f4d0dfa852, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733246191916 2024-12-03T17:16:33,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,876 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 798c3d2a8f0c4c36a06ca3863336ac87, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1733246188142 2024-12-03T17:16:33,877 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 019d440542c54f45b3d09dac09a5c53a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733246189399 2024-12-03T17:16:33,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,882 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29b6adb605054138af14279dc3c65489, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733246191916 2024-12-03T17:16:33,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-12-03T17:16:33,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-03T17:16:33,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,890 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T17:16:33,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,894 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T17:16:33,895 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T17:16:33,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,917 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e02d2d095200ac1f1a0b05c28cb936a#B#compaction#102 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:16:33,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,918 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/12f4fe45cc4e4a7b997fe059a4f0b3cc is 50, key is test_row_0/B:col10/1733246191916/Put/seqid=0 2024-12-03T17:16:33,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,923 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4e02d2d095200ac1f1a0b05c28cb936a] 2024-12-03T17:16:33,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,943 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203f2baf27255824d1fae40680f53467ccf_4e02d2d095200ac1f1a0b05c28cb936a store=[table=TestAcidGuarantees family=A region=4e02d2d095200ac1f1a0b05c28cb936a] 2024-12-03T17:16:33,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,948 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203f2baf27255824d1fae40680f53467ccf_4e02d2d095200ac1f1a0b05c28cb936a, store=[table=TestAcidGuarantees family=A region=4e02d2d095200ac1f1a0b05c28cb936a] 2024-12-03T17:16:33,948 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203f2baf27255824d1fae40680f53467ccf_4e02d2d095200ac1f1a0b05c28cb936a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4e02d2d095200ac1f1a0b05c28cb936a] 2024-12-03T17:16:33,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741949_1125 (size=12391) 2024-12-03T17:16:33,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,967 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/12f4fe45cc4e4a7b997fe059a4f0b3cc as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/12f4fe45cc4e4a7b997fe059a4f0b3cc 2024-12-03T17:16:33,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,983 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4e02d2d095200ac1f1a0b05c28cb936a/B of 4e02d2d095200ac1f1a0b05c28cb936a into 12f4fe45cc4e4a7b997fe059a4f0b3cc(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:16:33,983 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e02d2d095200ac1f1a0b05c28cb936a: 2024-12-03T17:16:33,983 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., storeName=4e02d2d095200ac1f1a0b05c28cb936a/B, priority=12, startTime=1733246193865; duration=0sec 2024-12-03T17:16:33,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,983 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:16:33,983 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e02d2d095200ac1f1a0b05c28cb936a:B 2024-12-03T17:16:33,983 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T17:16:33,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,985 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T17:16:33,985 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): 4e02d2d095200ac1f1a0b05c28cb936a/C is initiating minor compaction (all files) 2024-12-03T17:16:33,986 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e02d2d095200ac1f1a0b05c28cb936a/C in TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:33,986 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/e7b73b54d2924a0fa777fd318b512704, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/615be05b4f5345ec876cfc3805a0de45, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/0930543cebeb41f2af31f6b43422d954, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/ab4c98384c744c17842ee49a685b492f] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp, totalSize=47.1 K 2024-12-03T17:16:33,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,987 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting e7b73b54d2924a0fa777fd318b512704, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733246187988 2024-12-03T17:16:33,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,988 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 615be05b4f5345ec876cfc3805a0de45, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1733246188142 2024-12-03T17:16:33,988 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 0930543cebeb41f2af31f6b43422d954, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733246189399 2024-12-03T17:16:33,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,989 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting ab4c98384c744c17842ee49a685b492f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733246191916 2024-12-03T17:16:33,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:33,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-03T17:16:33,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741950_1126 (size=4469) 2024-12-03T17:16:34,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,008 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e02d2d095200ac1f1a0b05c28cb936a#C#compaction#104 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:16:34,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,012 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/2be9ea593b4245b38c7accc45dc49ba8 is 50, key is test_row_0/C:col10/1733246191916/Put/seqid=0 2024-12-03T17:16:34,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-03T17:16:34,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:34,058 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 4e02d2d095200ac1f1a0b05c28cb936a 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-03T17:16:34,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e02d2d095200ac1f1a0b05c28cb936a, store=A 2024-12-03T17:16:34,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:34,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e02d2d095200ac1f1a0b05c28cb936a, store=B 2024-12-03T17:16:34,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:34,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e02d2d095200ac1f1a0b05c28cb936a, store=C 2024-12-03T17:16:34,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:34,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741951_1127 (size=12391) 2024-12-03T17:16:34,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412035ad294dcef824b2e80949859344f0026_4e02d2d095200ac1f1a0b05c28cb936a is 50, key is test_row_0/A:col10/1733246192160/Put/seqid=0 2024-12-03T17:16:34,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,121 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/2be9ea593b4245b38c7accc45dc49ba8 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/2be9ea593b4245b38c7accc45dc49ba8 2024-12-03T17:16:34,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741952_1128 (size=9814) 2024-12-03T17:16:34,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,166 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4e02d2d095200ac1f1a0b05c28cb936a/C of 4e02d2d095200ac1f1a0b05c28cb936a into 2be9ea593b4245b38c7accc45dc49ba8(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:16:34,166 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e02d2d095200ac1f1a0b05c28cb936a: 2024-12-03T17:16:34,167 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., storeName=4e02d2d095200ac1f1a0b05c28cb936a/C, priority=12, startTime=1733246193867; duration=0sec 2024-12-03T17:16:34,167 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:34,167 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e02d2d095200ac1f1a0b05c28cb936a:C 2024-12-03T17:16:34,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-03T17:16:34,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:34,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. as already flushing 2024-12-03T17:16:34,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,416 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e02d2d095200ac1f1a0b05c28cb936a#A#compaction#103 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:16:34,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,418 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/0c678eddcf3c43d2a5fe449edfb6426f is 175, key is test_row_0/A:col10/1733246191916/Put/seqid=0 2024-12-03T17:16:34,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741953_1129 (size=31345) 2024-12-03T17:16:34,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,499 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/0c678eddcf3c43d2a5fe449edfb6426f as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/0c678eddcf3c43d2a5fe449edfb6426f 2024-12-03T17:16:34,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-03T17:16:34,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,531 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4e02d2d095200ac1f1a0b05c28cb936a/A of 4e02d2d095200ac1f1a0b05c28cb936a into 0c678eddcf3c43d2a5fe449edfb6426f(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:16:34,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,531 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e02d2d095200ac1f1a0b05c28cb936a: 2024-12-03T17:16:34,532 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., storeName=4e02d2d095200ac1f1a0b05c28cb936a/A, priority=12, startTime=1733246193865; duration=0sec 2024-12-03T17:16:34,532 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:34,532 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e02d2d095200ac1f1a0b05c28cb936a:A 2024-12-03T17:16:34,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,579 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412035ad294dcef824b2e80949859344f0026_4e02d2d095200ac1f1a0b05c28cb936a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412035ad294dcef824b2e80949859344f0026_4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:34,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/940c0724f2d749c1beec268e057a641e, store: [table=TestAcidGuarantees family=A region=4e02d2d095200ac1f1a0b05c28cb936a] 2024-12-03T17:16:34,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/940c0724f2d749c1beec268e057a641e is 175, key is test_row_0/A:col10/1733246192160/Put/seqid=0 2024-12-03T17:16:34,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,632 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733246194443, max timestamp=1733246194623Num uniques 859; 2024-12-03T17:16:34,633 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T17:16:34,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741954_1130 (size=22461) 2024-12-03T17:16:34,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,636 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733246194443, max timestamp=1733246194623Num uniques 859; 2024-12-03T17:16:34,636 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T17:16:34,637 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733246194443, max timestamp=1733246194623Num uniques 859; 2024-12-03T17:16:34,637 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T17:16:34,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:34,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:34,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246254771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:34,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:34,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:34,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:34,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246254779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:34,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:34,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:34,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:34,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:34,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:34,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246254793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:34,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:34,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246254793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:34,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:34,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:34,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:34,816 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:34,816 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:34,816 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:34,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:34,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246254906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:34,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:34,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:34,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:34,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:34,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246254906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:34,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:34,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:34,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:34,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:34,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246254914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:34,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:34,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:34,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:34,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:34,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246254923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:34,927 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:34,927 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:34,927 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:35,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-03T17:16:35,048 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=163, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/940c0724f2d749c1beec268e057a641e 2024-12-03T17:16:35,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/c1b5ade639fe459f96e6bc619b700106 is 50, key is test_row_0/B:col10/1733246192160/Put/seqid=0 2024-12-03T17:16:35,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741955_1131 (size=9757) 2024-12-03T17:16:35,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:35,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246255134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:35,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:35,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246255134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:35,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:35,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:35,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:35,145 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:35,145 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:35,145 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:35,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:35,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246255140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:35,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:35,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:35,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:35,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:35,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246255140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:35,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:35,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:35,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:35,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:35,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246255454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:35,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:35,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246255455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:35,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:35,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246255455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:35,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:35,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246255455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:35,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:35,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:35,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:35,466 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:35,466 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:35,466 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:35,466 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:35,466 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:35,466 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:35,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:35,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:35,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:35,492 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/c1b5ade639fe459f96e6bc619b700106 2024-12-03T17:16:35,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/e6ed175c84124febb1391041954099e8 is 50, key is test_row_0/C:col10/1733246192160/Put/seqid=0 2024-12-03T17:16:35,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741956_1132 (size=9757) 2024-12-03T17:16:35,563 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/e6ed175c84124febb1391041954099e8 2024-12-03T17:16:35,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/940c0724f2d749c1beec268e057a641e as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/940c0724f2d749c1beec268e057a641e 2024-12-03T17:16:35,622 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/940c0724f2d749c1beec268e057a641e, entries=100, sequenceid=163, filesize=21.9 K 2024-12-03T17:16:35,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/c1b5ade639fe459f96e6bc619b700106 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/c1b5ade639fe459f96e6bc619b700106 2024-12-03T17:16:35,640 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/c1b5ade639fe459f96e6bc619b700106, entries=100, sequenceid=163, filesize=9.5 K 2024-12-03T17:16:35,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/e6ed175c84124febb1391041954099e8 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/e6ed175c84124febb1391041954099e8 2024-12-03T17:16:35,671 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/e6ed175c84124febb1391041954099e8, entries=100, sequenceid=163, filesize=9.5 K 2024-12-03T17:16:35,673 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=147.60 KB/151140 for 4e02d2d095200ac1f1a0b05c28cb936a in 1614ms, sequenceid=163, compaction requested=false 2024-12-03T17:16:35,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 4e02d2d095200ac1f1a0b05c28cb936a: 2024-12-03T17:16:35,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:35,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-12-03T17:16:35,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-12-03T17:16:35,682 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-12-03T17:16:35,682 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7830 sec 2024-12-03T17:16:35,686 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 1.8070 sec 2024-12-03T17:16:35,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:35,984 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4e02d2d095200ac1f1a0b05c28cb936a 3/3 column families, dataSize=154.31 KB heapSize=430.69 KB 2024-12-03T17:16:35,989 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733246194746, max timestamp=1733246195982Num uniques 1168; 2024-12-03T17:16:35,989 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=9978, suffix heap size=149440,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=218 2024-12-03T17:16:35,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e02d2d095200ac1f1a0b05c28cb936a, store=A 2024-12-03T17:16:35,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T17:16:35,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e02d2d095200ac1f1a0b05c28cb936a, store=B 2024-12-03T17:16:35,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:35,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e02d2d095200ac1f1a0b05c28cb936a, store=C 2024-12-03T17:16:35,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T17:16:36,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203b92e6767fc1d40648443c81528062097_4e02d2d095200ac1f1a0b05c28cb936a is 50, key is test_row_0/A:col10/1733246194770/Put/seqid=0 2024-12-03T17:16:36,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-03T17:16:36,035 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:16:36,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:36,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246256023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:36,035 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T17:16:36,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:36,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:36,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:36,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:36,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246256025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:36,037 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:36,037 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:36,037 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:36,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:36,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246256028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:36,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:36,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:36,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:36,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:36,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246256028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:36,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-12-03T17:16:36,040 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:36,040 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:36,040 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:36,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741957_1133 (size=17284) 2024-12-03T17:16:36,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-03T17:16:36,042 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:36,043 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T17:16:36,044 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T17:16:36,044 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T17:16:36,052 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203b92e6767fc1d40648443c81528062097_4e02d2d095200ac1f1a0b05c28cb936a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b92e6767fc1d40648443c81528062097_4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:36,053 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/2f5a1b9d577f400d853eb283a3b297a0, store: [table=TestAcidGuarantees family=A region=4e02d2d095200ac1f1a0b05c28cb936a] 2024-12-03T17:16:36,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/2f5a1b9d577f400d853eb283a3b297a0 is 175, key is test_row_0/A:col10/1733246194770/Put/seqid=0 2024-12-03T17:16:36,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741958_1134 (size=48389) 2024-12-03T17:16:36,103 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=203, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/2f5a1b9d577f400d853eb283a3b297a0 2024-12-03T17:16:36,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/9f24770ba17f49a19f07a04cbd04e7c7 is 50, key is test_row_0/B:col10/1733246194770/Put/seqid=0 2024-12-03T17:16:36,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-03T17:16:36,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:36,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246256144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:36,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:36,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246256144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:36,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:36,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246256144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:36,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:36,157 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:36,157 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:36,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246256145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:36,157 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:36,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:36,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:36,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:36,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:36,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:36,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:36,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:36,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:36,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:36,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741959_1135 (size=12151) 2024-12-03T17:16:36,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-03T17:16:36,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:36,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. as already flushing 2024-12-03T17:16:36,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:36,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:36,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:36,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:36,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-03T17:16:36,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-03T17:16:36,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:36,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. as already flushing 2024-12-03T17:16:36,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:36,364 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:36,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:36,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:36,369 INFO [master/27e0f852a164:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-03T17:16:36,369 INFO [master/27e0f852a164:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-03T17:16:36,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:36,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:36,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:36,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246256365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:36,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246256364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:36,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246256365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:36,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:36,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246256367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:36,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:36,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:36,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:36,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:36,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:36,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:36,377 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:36,377 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:36,377 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:36,378 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:36,378 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:36,378 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:36,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-03T17:16:36,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:36,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. as already flushing 2024-12-03T17:16:36,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:36,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:36,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:36,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:36,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:36,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39046 deadline: 1733246256534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:36,547 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 27e0f852a164,41167,1733246151593 for 'test_row_0' in 4e02d2d095200ac1f1a0b05c28cb936a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8344 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T17:16:36,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:36,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:36,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:36,578 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/9f24770ba17f49a19f07a04cbd04e7c7 2024-12-03T17:16:36,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/3824d94afd544feb9d9e54a4c94412f8 is 50, key is test_row_0/C:col10/1733246194770/Put/seqid=0 2024-12-03T17:16:36,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741960_1136 (size=12151) 2024-12-03T17:16:36,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-03T17:16:36,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-03T17:16:36,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:36,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. as already flushing 2024-12-03T17:16:36,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:36,687 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:36,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:36,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:36,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:36,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246256685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:36,692 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:36,692 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:36,692 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:36,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:36,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246256685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:36,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:36,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246256685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:36,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:36,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:36,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:36,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:36,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:36,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:36,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:36,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246256685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:36,702 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:36,702 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:36,702 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:36,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-03T17:16:36,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:36,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. as already flushing 2024-12-03T17:16:36,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:36,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:36,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:36,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:37,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-03T17:16:37,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:37,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. as already flushing 2024-12-03T17:16:37,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:37,015 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:37,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:37,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:37,055 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/3824d94afd544feb9d9e54a4c94412f8 2024-12-03T17:16:37,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/2f5a1b9d577f400d853eb283a3b297a0 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/2f5a1b9d577f400d853eb283a3b297a0 2024-12-03T17:16:37,103 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/2f5a1b9d577f400d853eb283a3b297a0, entries=250, sequenceid=203, filesize=47.3 K 2024-12-03T17:16:37,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/9f24770ba17f49a19f07a04cbd04e7c7 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/9f24770ba17f49a19f07a04cbd04e7c7 2024-12-03T17:16:37,132 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/9f24770ba17f49a19f07a04cbd04e7c7, entries=150, sequenceid=203, filesize=11.9 K 2024-12-03T17:16:37,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/3824d94afd544feb9d9e54a4c94412f8 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/3824d94afd544feb9d9e54a4c94412f8 2024-12-03T17:16:37,148 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/3824d94afd544feb9d9e54a4c94412f8, entries=150, sequenceid=203, filesize=11.9 K 2024-12-03T17:16:37,152 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~118.53 KB/121370, heapSize ~327.70 KB/335568, currentSize=73.80 KB/75570 for 4e02d2d095200ac1f1a0b05c28cb936a in 1169ms, sequenceid=203, compaction requested=true 2024-12-03T17:16:37,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4e02d2d095200ac1f1a0b05c28cb936a: 2024-12-03T17:16:37,152 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:16:37,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e02d2d095200ac1f1a0b05c28cb936a:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T17:16:37,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:37,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e02d2d095200ac1f1a0b05c28cb936a:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T17:16:37,154 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:16:37,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:37,154 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:16:37,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e02d2d095200ac1f1a0b05c28cb936a:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T17:16:37,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:16:37,154 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): 4e02d2d095200ac1f1a0b05c28cb936a/A is initiating minor compaction (all files) 2024-12-03T17:16:37,154 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e02d2d095200ac1f1a0b05c28cb936a/A in TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:37,154 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/0c678eddcf3c43d2a5fe449edfb6426f, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/940c0724f2d749c1beec268e057a641e, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/2f5a1b9d577f400d853eb283a3b297a0] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp, totalSize=99.8 K 2024-12-03T17:16:37,154 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:37,154 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. files: [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/0c678eddcf3c43d2a5fe449edfb6426f, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/940c0724f2d749c1beec268e057a641e, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/2f5a1b9d577f400d853eb283a3b297a0] 2024-12-03T17:16:37,155 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c678eddcf3c43d2a5fe449edfb6426f, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733246191916 2024-12-03T17:16:37,156 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:16:37,156 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): 4e02d2d095200ac1f1a0b05c28cb936a/B is initiating minor compaction (all files) 2024-12-03T17:16:37,156 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e02d2d095200ac1f1a0b05c28cb936a/B in TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:37,156 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/12f4fe45cc4e4a7b997fe059a4f0b3cc, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/c1b5ade639fe459f96e6bc619b700106, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/9f24770ba17f49a19f07a04cbd04e7c7] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp, totalSize=33.5 K 2024-12-03T17:16:37,156 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 940c0724f2d749c1beec268e057a641e, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1733246192156 2024-12-03T17:16:37,156 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 12f4fe45cc4e4a7b997fe059a4f0b3cc, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733246191916 2024-12-03T17:16:37,157 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f5a1b9d577f400d853eb283a3b297a0, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733246194746 2024-12-03T17:16:37,157 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting c1b5ade639fe459f96e6bc619b700106, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1733246192156 2024-12-03T17:16:37,158 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f24770ba17f49a19f07a04cbd04e7c7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733246194770 2024-12-03T17:16:37,229 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4e02d2d095200ac1f1a0b05c28cb936a] 2024-12-03T17:16:37,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-03T17:16:37,232 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e02d2d095200ac1f1a0b05c28cb936a#B#compaction#112 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:16:37,232 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/1e7a7eab40ca4a9d99e186587aa3f782 is 50, key is test_row_0/B:col10/1733246194770/Put/seqid=0 2024-12-03T17:16:37,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:37,234 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 4e02d2d095200ac1f1a0b05c28cb936a 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-03T17:16:37,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e02d2d095200ac1f1a0b05c28cb936a, store=A 2024-12-03T17:16:37,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:37,235 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412034ea7e33e463e4eb882afa9b0d78e96c3_4e02d2d095200ac1f1a0b05c28cb936a store=[table=TestAcidGuarantees family=A region=4e02d2d095200ac1f1a0b05c28cb936a] 2024-12-03T17:16:37,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e02d2d095200ac1f1a0b05c28cb936a, store=B 2024-12-03T17:16:37,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:37,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e02d2d095200ac1f1a0b05c28cb936a, store=C 2024-12-03T17:16:37,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:37,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-03T17:16:37,237 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412034ea7e33e463e4eb882afa9b0d78e96c3_4e02d2d095200ac1f1a0b05c28cb936a, store=[table=TestAcidGuarantees family=A region=4e02d2d095200ac1f1a0b05c28cb936a] 2024-12-03T17:16:37,237 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412034ea7e33e463e4eb882afa9b0d78e96c3_4e02d2d095200ac1f1a0b05c28cb936a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4e02d2d095200ac1f1a0b05c28cb936a] 2024-12-03T17:16:37,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:37,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. as already flushing 2024-12-03T17:16:37,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:37,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246257287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:37,302 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:37,302 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:37,302 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:37,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:37,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246257299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:37,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:37,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246257300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:37,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:37,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246257302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:37,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:37,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:37,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:37,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:37,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:37,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:37,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:37,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:37,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:37,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412039350a5c31d4b48a5a2bb569ec0428095_4e02d2d095200ac1f1a0b05c28cb936a is 50, key is test_row_0/A:col10/1733246196021/Put/seqid=0 2024-12-03T17:16:37,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741962_1138 (size=4469) 2024-12-03T17:16:37,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741961_1137 (size=12493) 2024-12-03T17:16:37,354 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/1e7a7eab40ca4a9d99e186587aa3f782 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/1e7a7eab40ca4a9d99e186587aa3f782 2024-12-03T17:16:37,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741963_1139 (size=12304) 2024-12-03T17:16:37,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:37,380 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412039350a5c31d4b48a5a2bb569ec0428095_4e02d2d095200ac1f1a0b05c28cb936a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039350a5c31d4b48a5a2bb569ec0428095_4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:37,382 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4e02d2d095200ac1f1a0b05c28cb936a/B of 4e02d2d095200ac1f1a0b05c28cb936a into 1e7a7eab40ca4a9d99e186587aa3f782(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:16:37,382 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e02d2d095200ac1f1a0b05c28cb936a: 2024-12-03T17:16:37,382 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., storeName=4e02d2d095200ac1f1a0b05c28cb936a/B, priority=13, startTime=1733246197154; duration=0sec 2024-12-03T17:16:37,382 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:16:37,382 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e02d2d095200ac1f1a0b05c28cb936a:B 2024-12-03T17:16:37,382 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:16:37,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/1ac542e15ae0463d85f3f90192e4c444, store: [table=TestAcidGuarantees family=A region=4e02d2d095200ac1f1a0b05c28cb936a] 2024-12-03T17:16:37,394 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:16:37,394 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): 4e02d2d095200ac1f1a0b05c28cb936a/C is initiating minor compaction (all files) 2024-12-03T17:16:37,394 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e02d2d095200ac1f1a0b05c28cb936a/C in TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:37,394 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/2be9ea593b4245b38c7accc45dc49ba8, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/e6ed175c84124febb1391041954099e8, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/3824d94afd544feb9d9e54a4c94412f8] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp, totalSize=33.5 K 2024-12-03T17:16:37,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/1ac542e15ae0463d85f3f90192e4c444 is 175, key is test_row_0/A:col10/1733246196021/Put/seqid=0 2024-12-03T17:16:37,399 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 2be9ea593b4245b38c7accc45dc49ba8, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733246191916 2024-12-03T17:16:37,401 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting e6ed175c84124febb1391041954099e8, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1733246192156 2024-12-03T17:16:37,403 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 3824d94afd544feb9d9e54a4c94412f8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733246194770 2024-12-03T17:16:37,417 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e02d2d095200ac1f1a0b05c28cb936a#C#compaction#114 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:16:37,417 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/d72b9698989643eab4a1683ebe93a822 is 50, key is test_row_0/C:col10/1733246194770/Put/seqid=0 2024-12-03T17:16:37,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:37,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246257413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:37,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:37,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246257413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:37,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:37,425 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:37,425 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:37,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246257413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:37,425 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:37,425 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:37,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:37,425 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:37,425 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:37,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:37,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:37,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246257425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:37,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:37,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:37,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:37,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:37,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741964_1140 (size=31105) 2024-12-03T17:16:37,463 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=217, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/1ac542e15ae0463d85f3f90192e4c444 2024-12-03T17:16:37,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741965_1141 (size=12493) 2024-12-03T17:16:37,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/eb69f9610f36488c9f6facae35a458b5 is 50, key is test_row_0/B:col10/1733246196021/Put/seqid=0 2024-12-03T17:16:37,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741966_1142 (size=12151) 2024-12-03T17:16:37,550 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/eb69f9610f36488c9f6facae35a458b5 2024-12-03T17:16:37,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/1d7a810fa8274728a8ab77e759814941 is 50, key is test_row_0/C:col10/1733246196021/Put/seqid=0 2024-12-03T17:16:37,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741967_1143 (size=12151) 2024-12-03T17:16:37,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:37,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246257639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:37,641 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/1d7a810fa8274728a8ab77e759814941 2024-12-03T17:16:37,641 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:37,641 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:37,642 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:37,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:37,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246257639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:37,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:37,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:37,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246257639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:37,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246257639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:37,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:37,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:37,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:37,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:37,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:37,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:37,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:37,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:37,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:37,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/1ac542e15ae0463d85f3f90192e4c444 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/1ac542e15ae0463d85f3f90192e4c444 2024-12-03T17:16:37,680 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/1ac542e15ae0463d85f3f90192e4c444, entries=150, sequenceid=217, filesize=30.4 K 2024-12-03T17:16:37,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/eb69f9610f36488c9f6facae35a458b5 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/eb69f9610f36488c9f6facae35a458b5 2024-12-03T17:16:37,688 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/eb69f9610f36488c9f6facae35a458b5, entries=150, sequenceid=217, filesize=11.9 K 2024-12-03T17:16:37,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/1d7a810fa8274728a8ab77e759814941 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/1d7a810fa8274728a8ab77e759814941 2024-12-03T17:16:37,703 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/1d7a810fa8274728a8ab77e759814941, entries=150, sequenceid=217, filesize=11.9 K 2024-12-03T17:16:37,704 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 4e02d2d095200ac1f1a0b05c28cb936a in 470ms, sequenceid=217, compaction requested=false 2024-12-03T17:16:37,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 4e02d2d095200ac1f1a0b05c28cb936a: 2024-12-03T17:16:37,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:37,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-12-03T17:16:37,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-12-03T17:16:37,720 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-12-03T17:16:37,720 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6630 sec 2024-12-03T17:16:37,726 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.6860 sec 2024-12-03T17:16:37,728 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e02d2d095200ac1f1a0b05c28cb936a#A#compaction#111 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:16:37,729 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/77748662e1f84faeae78033999be84e0 is 175, key is test_row_0/A:col10/1733246194770/Put/seqid=0 2024-12-03T17:16:37,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741968_1144 (size=31447) 2024-12-03T17:16:37,762 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/77748662e1f84faeae78033999be84e0 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/77748662e1f84faeae78033999be84e0 2024-12-03T17:16:37,812 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4e02d2d095200ac1f1a0b05c28cb936a/A of 4e02d2d095200ac1f1a0b05c28cb936a into 77748662e1f84faeae78033999be84e0(size=30.7 K), total size for store is 61.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:16:37,812 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e02d2d095200ac1f1a0b05c28cb936a: 2024-12-03T17:16:37,812 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., storeName=4e02d2d095200ac1f1a0b05c28cb936a/A, priority=13, startTime=1733246197152; duration=0sec 2024-12-03T17:16:37,812 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:37,812 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e02d2d095200ac1f1a0b05c28cb936a:A 2024-12-03T17:16:37,931 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/d72b9698989643eab4a1683ebe93a822 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/d72b9698989643eab4a1683ebe93a822 2024-12-03T17:16:37,959 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4e02d2d095200ac1f1a0b05c28cb936a 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-03T17:16:37,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e02d2d095200ac1f1a0b05c28cb936a, store=A 2024-12-03T17:16:37,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:37,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e02d2d095200ac1f1a0b05c28cb936a, store=B 2024-12-03T17:16:37,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:37,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e02d2d095200ac1f1a0b05c28cb936a, store=C 2024-12-03T17:16:37,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:37,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:37,965 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4e02d2d095200ac1f1a0b05c28cb936a/C of 4e02d2d095200ac1f1a0b05c28cb936a into d72b9698989643eab4a1683ebe93a822(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:16:37,965 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e02d2d095200ac1f1a0b05c28cb936a: 2024-12-03T17:16:37,965 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., storeName=4e02d2d095200ac1f1a0b05c28cb936a/C, priority=13, startTime=1733246197154; duration=0sec 2024-12-03T17:16:37,966 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:37,966 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e02d2d095200ac1f1a0b05c28cb936a:C 2024-12-03T17:16:37,980 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203e4876b489c174d88a4e74fc7147b9840_4e02d2d095200ac1f1a0b05c28cb936a is 50, key is test_row_0/A:col10/1733246197298/Put/seqid=0 2024-12-03T17:16:37,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741969_1145 (size=12304) 2024-12-03T17:16:37,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:37,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246257988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:38,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:38,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:38,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246257988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:38,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:38,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:38,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:38,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246257989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:38,003 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:38,003 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:38,003 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:38,003 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:38,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:38,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:38,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:38,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:38,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246257992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:38,010 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:38,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:38,012 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:38,015 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203e4876b489c174d88a4e74fc7147b9840_4e02d2d095200ac1f1a0b05c28cb936a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203e4876b489c174d88a4e74fc7147b9840_4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:38,022 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/5a479282410143cba911c6d295dbe23f, store: [table=TestAcidGuarantees family=A region=4e02d2d095200ac1f1a0b05c28cb936a] 2024-12-03T17:16:38,023 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/5a479282410143cba911c6d295dbe23f is 175, key is test_row_0/A:col10/1733246197298/Put/seqid=0 2024-12-03T17:16:38,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741970_1146 (size=31105) 2024-12-03T17:16:38,069 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=238, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/5a479282410143cba911c6d295dbe23f 2024-12-03T17:16:38,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/e268adbe098b4bfba6a8b7d49320107d is 50, key is test_row_0/B:col10/1733246197298/Put/seqid=0 2024-12-03T17:16:38,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:38,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246258114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:38,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:38,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246258114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:38,122 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:38,122 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:38,122 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:38,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:38,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246258114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:38,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:38,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246258114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:38,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:38,124 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:38,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:38,124 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:38,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:38,124 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:38,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:38,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:38,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:38,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741971_1147 (size=12151) 2024-12-03T17:16:38,153 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/e268adbe098b4bfba6a8b7d49320107d 2024-12-03T17:16:38,201 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/3de3e35a453a48918cfaf057c6b0a124 is 50, key is test_row_0/C:col10/1733246197298/Put/seqid=0 2024-12-03T17:16:38,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741972_1148 (size=12151) 2024-12-03T17:16:38,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-03T17:16:38,255 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:16:38,258 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T17:16:38,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-12-03T17:16:38,261 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T17:16:38,264 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T17:16:38,264 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T17:16:38,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-03T17:16:38,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:38,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246258333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:38,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:38,338 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:38,338 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:38,338 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:38,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:38,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246258334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:38,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:38,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246258334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:38,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:38,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:38,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:38,340 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:38,340 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:38,340 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:38,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246258334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:38,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:38,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:38,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:38,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-03T17:16:38,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-03T17:16:38,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:38,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. as already flushing 2024-12-03T17:16:38,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:38,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:38,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:38,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:38,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-03T17:16:38,591 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-03T17:16:38,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:38,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. as already flushing 2024-12-03T17:16:38,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:38,595 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:38,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:38,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:38,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:38,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246258645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:38,646 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/3de3e35a453a48918cfaf057c6b0a124 2024-12-03T17:16:38,646 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:38,646 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:38,646 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:38,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:38,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246258645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:38,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:38,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246258645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:38,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:38,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:38,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:38,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:38,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:38,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:38,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:38,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246258655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:38,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:38,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:38,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:38,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/5a479282410143cba911c6d295dbe23f as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/5a479282410143cba911c6d295dbe23f 2024-12-03T17:16:38,691 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/5a479282410143cba911c6d295dbe23f, entries=150, sequenceid=238, filesize=30.4 K 2024-12-03T17:16:38,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/e268adbe098b4bfba6a8b7d49320107d as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/e268adbe098b4bfba6a8b7d49320107d 2024-12-03T17:16:38,719 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/e268adbe098b4bfba6a8b7d49320107d, entries=150, sequenceid=238, filesize=11.9 K 2024-12-03T17:16:38,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/3de3e35a453a48918cfaf057c6b0a124 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/3de3e35a453a48918cfaf057c6b0a124 2024-12-03T17:16:38,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-03T17:16:38,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:38,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. as already flushing 2024-12-03T17:16:38,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:38,757 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:38,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:38,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:38,770 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/3de3e35a453a48918cfaf057c6b0a124, entries=150, sequenceid=238, filesize=11.9 K 2024-12-03T17:16:38,778 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 4e02d2d095200ac1f1a0b05c28cb936a in 819ms, sequenceid=238, compaction requested=true 2024-12-03T17:16:38,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4e02d2d095200ac1f1a0b05c28cb936a: 2024-12-03T17:16:38,778 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:16:38,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e02d2d095200ac1f1a0b05c28cb936a:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T17:16:38,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:38,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e02d2d095200ac1f1a0b05c28cb936a:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T17:16:38,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:16:38,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e02d2d095200ac1f1a0b05c28cb936a:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T17:16:38,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T17:16:38,782 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:16:38,798 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93657 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:16:38,798 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): 4e02d2d095200ac1f1a0b05c28cb936a/A is initiating minor compaction (all files) 2024-12-03T17:16:38,798 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e02d2d095200ac1f1a0b05c28cb936a/A in TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:38,798 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/77748662e1f84faeae78033999be84e0, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/1ac542e15ae0463d85f3f90192e4c444, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/5a479282410143cba911c6d295dbe23f] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp, totalSize=91.5 K 2024-12-03T17:16:38,798 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:38,798 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. files: [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/77748662e1f84faeae78033999be84e0, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/1ac542e15ae0463d85f3f90192e4c444, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/5a479282410143cba911c6d295dbe23f] 2024-12-03T17:16:38,807 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 77748662e1f84faeae78033999be84e0, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733246194770 2024-12-03T17:16:38,809 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:16:38,809 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): 4e02d2d095200ac1f1a0b05c28cb936a/B is initiating minor compaction (all files) 2024-12-03T17:16:38,810 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e02d2d095200ac1f1a0b05c28cb936a/B in TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:38,810 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/1e7a7eab40ca4a9d99e186587aa3f782, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/eb69f9610f36488c9f6facae35a458b5, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/e268adbe098b4bfba6a8b7d49320107d] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp, totalSize=35.9 K 2024-12-03T17:16:38,811 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ac542e15ae0463d85f3f90192e4c444, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733246195998 2024-12-03T17:16:38,811 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e7a7eab40ca4a9d99e186587aa3f782, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733246194770 2024-12-03T17:16:38,819 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb69f9610f36488c9f6facae35a458b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733246195998 2024-12-03T17:16:38,819 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a479282410143cba911c6d295dbe23f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733246197285 2024-12-03T17:16:38,827 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting e268adbe098b4bfba6a8b7d49320107d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733246197285 2024-12-03T17:16:38,891 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4e02d2d095200ac1f1a0b05c28cb936a] 2024-12-03T17:16:38,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-03T17:16:38,905 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e02d2d095200ac1f1a0b05c28cb936a#B#compaction#121 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:16:38,906 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/fdb16c78fc9740d2b579c1ed33b60c6b is 50, key is test_row_0/B:col10/1733246197298/Put/seqid=0 2024-12-03T17:16:38,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-03T17:16:38,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:38,923 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 4e02d2d095200ac1f1a0b05c28cb936a 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T17:16:38,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e02d2d095200ac1f1a0b05c28cb936a, store=A 2024-12-03T17:16:38,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:38,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e02d2d095200ac1f1a0b05c28cb936a, store=B 2024-12-03T17:16:38,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:38,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e02d2d095200ac1f1a0b05c28cb936a, store=C 2024-12-03T17:16:38,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:38,931 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120393701baf4d4b4ba7a628e976eafc03e9_4e02d2d095200ac1f1a0b05c28cb936a store=[table=TestAcidGuarantees family=A region=4e02d2d095200ac1f1a0b05c28cb936a] 2024-12-03T17:16:38,933 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120393701baf4d4b4ba7a628e976eafc03e9_4e02d2d095200ac1f1a0b05c28cb936a, store=[table=TestAcidGuarantees family=A region=4e02d2d095200ac1f1a0b05c28cb936a] 2024-12-03T17:16:38,933 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120393701baf4d4b4ba7a628e976eafc03e9_4e02d2d095200ac1f1a0b05c28cb936a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4e02d2d095200ac1f1a0b05c28cb936a] 2024-12-03T17:16:38,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741973_1149 (size=12595) 2024-12-03T17:16:38,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ecc1dd994939412db895a41798347a3b_4e02d2d095200ac1f1a0b05c28cb936a is 50, key is test_row_0/A:col10/1733246197983/Put/seqid=0 2024-12-03T17:16:38,998 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/fdb16c78fc9740d2b579c1ed33b60c6b as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/fdb16c78fc9740d2b579c1ed33b60c6b 2024-12-03T17:16:39,015 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4e02d2d095200ac1f1a0b05c28cb936a/B of 4e02d2d095200ac1f1a0b05c28cb936a into fdb16c78fc9740d2b579c1ed33b60c6b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:16:39,015 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e02d2d095200ac1f1a0b05c28cb936a: 2024-12-03T17:16:39,015 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., storeName=4e02d2d095200ac1f1a0b05c28cb936a/B, priority=13, startTime=1733246198782; duration=0sec 2024-12-03T17:16:39,015 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:16:39,015 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e02d2d095200ac1f1a0b05c28cb936a:B 2024-12-03T17:16:39,015 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:16:39,018 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:16:39,018 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): 4e02d2d095200ac1f1a0b05c28cb936a/C is initiating minor compaction (all files) 2024-12-03T17:16:39,018 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e02d2d095200ac1f1a0b05c28cb936a/C in TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:39,018 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/d72b9698989643eab4a1683ebe93a822, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/1d7a810fa8274728a8ab77e759814941, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/3de3e35a453a48918cfaf057c6b0a124] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp, totalSize=35.9 K 2024-12-03T17:16:39,018 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting d72b9698989643eab4a1683ebe93a822, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733246194770 2024-12-03T17:16:39,019 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d7a810fa8274728a8ab77e759814941, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733246195998 2024-12-03T17:16:39,019 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3de3e35a453a48918cfaf057c6b0a124, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733246197285 2024-12-03T17:16:39,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741974_1150 (size=4469) 2024-12-03T17:16:39,034 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e02d2d095200ac1f1a0b05c28cb936a#A#compaction#120 average throughput is 0.17 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:16:39,035 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/e72595ba7f88482e8be46fed73e77854 is 175, key is test_row_0/A:col10/1733246197298/Put/seqid=0 2024-12-03T17:16:39,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741975_1151 (size=12304) 2024-12-03T17:16:39,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:39,046 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e02d2d095200ac1f1a0b05c28cb936a#C#compaction#123 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:16:39,047 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/141c92d5224f43f49ee78788011d3d78 is 50, key is test_row_0/C:col10/1733246197298/Put/seqid=0 2024-12-03T17:16:39,052 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ecc1dd994939412db895a41798347a3b_4e02d2d095200ac1f1a0b05c28cb936a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ecc1dd994939412db895a41798347a3b_4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:39,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/3391385ac8c040deb36c4c56ab6b68dd, store: [table=TestAcidGuarantees family=A region=4e02d2d095200ac1f1a0b05c28cb936a] 2024-12-03T17:16:39,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/3391385ac8c040deb36c4c56ab6b68dd is 175, key is test_row_0/A:col10/1733246197983/Put/seqid=0 2024-12-03T17:16:39,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741976_1152 (size=31549) 2024-12-03T17:16:39,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741978_1154 (size=31105) 2024-12-03T17:16:39,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741977_1153 (size=12595) 2024-12-03T17:16:39,119 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/e72595ba7f88482e8be46fed73e77854 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/e72595ba7f88482e8be46fed73e77854 2024-12-03T17:16:39,138 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4e02d2d095200ac1f1a0b05c28cb936a/A of 4e02d2d095200ac1f1a0b05c28cb936a into e72595ba7f88482e8be46fed73e77854(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:16:39,138 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e02d2d095200ac1f1a0b05c28cb936a: 2024-12-03T17:16:39,138 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., storeName=4e02d2d095200ac1f1a0b05c28cb936a/A, priority=13, startTime=1733246198778; duration=0sec 2024-12-03T17:16:39,138 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:39,139 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e02d2d095200ac1f1a0b05c28cb936a:A 2024-12-03T17:16:39,147 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/141c92d5224f43f49ee78788011d3d78 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/141c92d5224f43f49ee78788011d3d78 2024-12-03T17:16:39,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:39,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. as already flushing 2024-12-03T17:16:39,158 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4e02d2d095200ac1f1a0b05c28cb936a/C of 4e02d2d095200ac1f1a0b05c28cb936a into 141c92d5224f43f49ee78788011d3d78(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:16:39,158 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e02d2d095200ac1f1a0b05c28cb936a: 2024-12-03T17:16:39,159 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., storeName=4e02d2d095200ac1f1a0b05c28cb936a/C, priority=13, startTime=1733246198782; duration=0sec 2024-12-03T17:16:39,159 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:39,159 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e02d2d095200ac1f1a0b05c28cb936a:C 2024-12-03T17:16:39,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:39,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246259213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:39,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:39,228 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:39,228 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:39,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246259215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:39,228 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:39,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:39,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:39,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:39,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:39,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246259225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:39,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:39,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:39,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:39,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:39,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246259228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:39,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:39,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:39,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:39,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:39,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246259335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:39,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:39,344 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:39,344 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:39,344 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:39,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246259335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:39,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:39,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246259343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:39,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:39,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246259343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:39,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:39,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:39,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:39,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:39,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:39,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:39,348 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:39,348 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:39,348 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:39,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-03T17:16:39,498 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=252, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/3391385ac8c040deb36c4c56ab6b68dd 2024-12-03T17:16:39,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/409beb3dd0c040ed914961b69a3d7341 is 50, key is test_row_0/B:col10/1733246197983/Put/seqid=0 2024-12-03T17:16:39,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:39,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246259554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:39,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741979_1155 (size=12151) 2024-12-03T17:16:39,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:39,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246259554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:39,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:39,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:39,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:39,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:39,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246259554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:39,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:39,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:39,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246259554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:39,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:39,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:39,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:39,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:39,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:39,562 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:39,562 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:39,562 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:39,562 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/409beb3dd0c040ed914961b69a3d7341 2024-12-03T17:16:39,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/40c9ca9405954afe92fb4b34ce5737ea is 50, key is test_row_0/C:col10/1733246197983/Put/seqid=0 2024-12-03T17:16:39,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741980_1156 (size=12151) 2024-12-03T17:16:39,609 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/40c9ca9405954afe92fb4b34ce5737ea 2024-12-03T17:16:39,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/3391385ac8c040deb36c4c56ab6b68dd as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/3391385ac8c040deb36c4c56ab6b68dd 2024-12-03T17:16:39,631 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/3391385ac8c040deb36c4c56ab6b68dd, entries=150, sequenceid=252, filesize=30.4 K 2024-12-03T17:16:39,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/409beb3dd0c040ed914961b69a3d7341 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/409beb3dd0c040ed914961b69a3d7341 2024-12-03T17:16:39,649 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/409beb3dd0c040ed914961b69a3d7341, entries=150, sequenceid=252, filesize=11.9 K 2024-12-03T17:16:39,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/40c9ca9405954afe92fb4b34ce5737ea as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/40c9ca9405954afe92fb4b34ce5737ea 2024-12-03T17:16:39,657 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/40c9ca9405954afe92fb4b34ce5737ea, entries=150, sequenceid=252, filesize=11.9 K 2024-12-03T17:16:39,658 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 4e02d2d095200ac1f1a0b05c28cb936a in 735ms, sequenceid=252, compaction requested=false 2024-12-03T17:16:39,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 4e02d2d095200ac1f1a0b05c28cb936a: 2024-12-03T17:16:39,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:39,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-12-03T17:16:39,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-12-03T17:16:39,665 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-12-03T17:16:39,665 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3990 sec 2024-12-03T17:16:39,667 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.4070 sec 2024-12-03T17:16:39,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:39,868 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4e02d2d095200ac1f1a0b05c28cb936a 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T17:16:39,885 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733246199196, max timestamp=1733246199867Num uniques 859; 2024-12-03T17:16:39,885 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T17:16:39,885 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733246199196, max timestamp=1733246199867Num uniques 859; 2024-12-03T17:16:39,885 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T17:16:39,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e02d2d095200ac1f1a0b05c28cb936a, store=A 2024-12-03T17:16:39,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:39,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e02d2d095200ac1f1a0b05c28cb936a, store=B 2024-12-03T17:16:39,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:39,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e02d2d095200ac1f1a0b05c28cb936a, store=C 2024-12-03T17:16:39,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:39,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412036c57e851c5f34b928a2402b4fdbabf26_4e02d2d095200ac1f1a0b05c28cb936a is 50, key is test_row_0/A:col10/1733246199196/Put/seqid=0 2024-12-03T17:16:39,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741981_1157 (size=12454) 2024-12-03T17:16:39,948 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:39,953 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733246199926, max timestamp=1733246199936Num uniques 859; 2024-12-03T17:16:39,953 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T17:16:39,956 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733246199926, max timestamp=1733246199936Num uniques 859; 2024-12-03T17:16:39,956 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T17:16:39,959 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733246199926, max timestamp=1733246199936Num uniques 859; 2024-12-03T17:16:39,960 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-03T17:16:39,980 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412036c57e851c5f34b928a2402b4fdbabf26_4e02d2d095200ac1f1a0b05c28cb936a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036c57e851c5f34b928a2402b4fdbabf26_4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:39,993 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/13543fc2a6454c5481e80581e87e091d, store: [table=TestAcidGuarantees family=A region=4e02d2d095200ac1f1a0b05c28cb936a] 2024-12-03T17:16:39,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/13543fc2a6454c5481e80581e87e091d is 175, key is test_row_0/A:col10/1733246199196/Put/seqid=0 2024-12-03T17:16:40,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:40,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246260006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:40,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:40,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:40,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:40,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:40,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246260023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:40,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:40,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246260027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:40,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:40,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:40,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:40,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741982_1158 (size=31255) 2024-12-03T17:16:40,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:40,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246260029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:40,036 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:40,036 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:40,036 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:40,037 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:40,037 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:40,037 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:40,050 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=276, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/13543fc2a6454c5481e80581e87e091d 2024-12-03T17:16:40,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/e7e0b84ff17e438fad92e90b8e2607bb is 50, key is test_row_0/B:col10/1733246199196/Put/seqid=0 2024-12-03T17:16:40,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:40,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246260136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:40,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:40,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:40,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:40,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:40,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246260144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:40,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:40,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:40,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:40,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:40,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246260144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:40,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:40,154 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:40,154 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:40,154 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:40,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246260145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:40,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741983_1159 (size=12301) 2024-12-03T17:16:40,159 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:40,159 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:40,159 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:40,162 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/e7e0b84ff17e438fad92e90b8e2607bb 2024-12-03T17:16:40,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/44a45684da3841ce8b389d9a1bb25b57 is 50, key is test_row_0/C:col10/1733246199196/Put/seqid=0 2024-12-03T17:16:40,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741984_1160 (size=12301) 2024-12-03T17:16:40,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:40,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246260344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:40,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:40,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:40,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:40,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:40,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246260354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:40,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:40,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:40,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:40,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:40,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246260371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:40,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:40,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246260371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:40,380 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:40,380 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:40,380 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:40,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:40,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:40,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:40,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-03T17:16:40,424 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:16:40,425 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T17:16:40,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-12-03T17:16:40,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-03T17:16:40,440 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T17:16:40,445 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T17:16:40,445 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T17:16:40,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-03T17:16:40,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-03T17:16:40,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:40,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. as already flushing 2024-12-03T17:16:40,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:40,606 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:40,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:40,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:40,662 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/44a45684da3841ce8b389d9a1bb25b57 2024-12-03T17:16:40,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:40,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246260664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:40,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:40,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:40,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:40,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:40,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246260678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:40,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:40,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:40,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:40,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:40,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:40,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246260694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:40,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/13543fc2a6454c5481e80581e87e091d as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/13543fc2a6454c5481e80581e87e091d 2024-12-03T17:16:40,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:40,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:40,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:40,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246260685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:40,699 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:40,699 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:40,699 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:40,739 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/13543fc2a6454c5481e80581e87e091d, entries=150, sequenceid=276, filesize=30.5 K 2024-12-03T17:16:40,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/e7e0b84ff17e438fad92e90b8e2607bb as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/e7e0b84ff17e438fad92e90b8e2607bb 2024-12-03T17:16:40,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-03T17:16:40,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-03T17:16:40,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:40,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. as already flushing 2024-12-03T17:16:40,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:40,774 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:40,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:40,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:40,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/e7e0b84ff17e438fad92e90b8e2607bb, entries=150, sequenceid=276, filesize=12.0 K 2024-12-03T17:16:40,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/44a45684da3841ce8b389d9a1bb25b57 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/44a45684da3841ce8b389d9a1bb25b57 2024-12-03T17:16:40,817 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/44a45684da3841ce8b389d9a1bb25b57, entries=150, sequenceid=276, filesize=12.0 K 2024-12-03T17:16:40,821 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~58.14 KB/59540, heapSize ~164.13 KB/168064, currentSize=127.08 KB/130125 for 4e02d2d095200ac1f1a0b05c28cb936a in 952ms, sequenceid=276, compaction requested=true 2024-12-03T17:16:40,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4e02d2d095200ac1f1a0b05c28cb936a: 2024-12-03T17:16:40,821 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:16:40,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e02d2d095200ac1f1a0b05c28cb936a:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T17:16:40,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:40,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e02d2d095200ac1f1a0b05c28cb936a:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T17:16:40,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:16:40,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e02d2d095200ac1f1a0b05c28cb936a:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T17:16:40,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T17:16:40,826 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:16:40,827 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93909 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:16:40,827 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): 4e02d2d095200ac1f1a0b05c28cb936a/A is initiating minor compaction (all files) 2024-12-03T17:16:40,827 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e02d2d095200ac1f1a0b05c28cb936a/A in TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:40,827 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/e72595ba7f88482e8be46fed73e77854, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/3391385ac8c040deb36c4c56ab6b68dd, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/13543fc2a6454c5481e80581e87e091d] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp, totalSize=91.7 K 2024-12-03T17:16:40,827 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:40,828 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. files: [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/e72595ba7f88482e8be46fed73e77854, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/3391385ac8c040deb36c4c56ab6b68dd, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/13543fc2a6454c5481e80581e87e091d] 2024-12-03T17:16:40,833 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting e72595ba7f88482e8be46fed73e77854, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733246197285 2024-12-03T17:16:40,835 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:16:40,835 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): 4e02d2d095200ac1f1a0b05c28cb936a/B is initiating minor compaction (all files) 2024-12-03T17:16:40,835 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e02d2d095200ac1f1a0b05c28cb936a/B in TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:40,835 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/fdb16c78fc9740d2b579c1ed33b60c6b, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/409beb3dd0c040ed914961b69a3d7341, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/e7e0b84ff17e438fad92e90b8e2607bb] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp, totalSize=36.2 K 2024-12-03T17:16:40,836 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 3391385ac8c040deb36c4c56ab6b68dd, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1733246197983 2024-12-03T17:16:40,839 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 13543fc2a6454c5481e80581e87e091d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733246199196 2024-12-03T17:16:40,839 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting fdb16c78fc9740d2b579c1ed33b60c6b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733246197285 2024-12-03T17:16:40,840 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 409beb3dd0c040ed914961b69a3d7341, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1733246197983 2024-12-03T17:16:40,841 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7e0b84ff17e438fad92e90b8e2607bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733246199196 2024-12-03T17:16:40,856 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4e02d2d095200ac1f1a0b05c28cb936a] 2024-12-03T17:16:40,876 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203c55d83267efc4e8ab83ea77e6905c2a6_4e02d2d095200ac1f1a0b05c28cb936a store=[table=TestAcidGuarantees family=A region=4e02d2d095200ac1f1a0b05c28cb936a] 2024-12-03T17:16:40,876 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e02d2d095200ac1f1a0b05c28cb936a#B#compaction#130 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:16:40,877 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/e84ca4b239cb4f5e9ce06b2707a67900 is 50, key is test_row_0/B:col10/1733246199196/Put/seqid=0 2024-12-03T17:16:40,878 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203c55d83267efc4e8ab83ea77e6905c2a6_4e02d2d095200ac1f1a0b05c28cb936a, store=[table=TestAcidGuarantees family=A region=4e02d2d095200ac1f1a0b05c28cb936a] 2024-12-03T17:16:40,878 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203c55d83267efc4e8ab83ea77e6905c2a6_4e02d2d095200ac1f1a0b05c28cb936a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4e02d2d095200ac1f1a0b05c28cb936a] 2024-12-03T17:16:40,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741985_1161 (size=12847) 2024-12-03T17:16:40,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741986_1162 (size=4469) 2024-12-03T17:16:40,936 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e02d2d095200ac1f1a0b05c28cb936a#A#compaction#129 average throughput is 0.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:16:40,936 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/597dbc88ef704d0a8a736e149f091dd4 is 175, key is test_row_0/A:col10/1733246199196/Put/seqid=0 2024-12-03T17:16:40,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-03T17:16:40,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:40,946 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 4e02d2d095200ac1f1a0b05c28cb936a 3/3 column families, dataSize=127.08 KB heapSize=364.15 KB 2024-12-03T17:16:40,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e02d2d095200ac1f1a0b05c28cb936a, store=A 2024-12-03T17:16:40,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:40,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e02d2d095200ac1f1a0b05c28cb936a, store=B 2024-12-03T17:16:40,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:40,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e02d2d095200ac1f1a0b05c28cb936a, store=C 2024-12-03T17:16:40,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:40,952 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/e84ca4b239cb4f5e9ce06b2707a67900 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/e84ca4b239cb4f5e9ce06b2707a67900 2024-12-03T17:16:40,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741987_1163 (size=31801) 2024-12-03T17:16:40,969 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4e02d2d095200ac1f1a0b05c28cb936a/B of 4e02d2d095200ac1f1a0b05c28cb936a into e84ca4b239cb4f5e9ce06b2707a67900(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:16:40,969 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e02d2d095200ac1f1a0b05c28cb936a: 2024-12-03T17:16:40,969 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., storeName=4e02d2d095200ac1f1a0b05c28cb936a/B, priority=13, startTime=1733246200823; duration=0sec 2024-12-03T17:16:40,969 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:16:40,969 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e02d2d095200ac1f1a0b05c28cb936a:B 2024-12-03T17:16:40,969 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:16:40,971 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:16:40,971 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): 4e02d2d095200ac1f1a0b05c28cb936a/C is initiating minor compaction (all files) 2024-12-03T17:16:40,971 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e02d2d095200ac1f1a0b05c28cb936a/C in TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:40,972 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/141c92d5224f43f49ee78788011d3d78, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/40c9ca9405954afe92fb4b34ce5737ea, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/44a45684da3841ce8b389d9a1bb25b57] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp, totalSize=36.2 K 2024-12-03T17:16:40,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203b6c853f3801749b6b08d973d3b6cb932_4e02d2d095200ac1f1a0b05c28cb936a is 50, key is test_row_0/A:col10/1733246200009/Put/seqid=0 2024-12-03T17:16:40,973 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 141c92d5224f43f49ee78788011d3d78, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733246197285 2024-12-03T17:16:40,980 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40c9ca9405954afe92fb4b34ce5737ea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1733246197983 2024-12-03T17:16:40,982 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44a45684da3841ce8b389d9a1bb25b57, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733246199196 2024-12-03T17:16:40,986 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/597dbc88ef704d0a8a736e149f091dd4 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/597dbc88ef704d0a8a736e149f091dd4 2024-12-03T17:16:40,996 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4e02d2d095200ac1f1a0b05c28cb936a/A of 4e02d2d095200ac1f1a0b05c28cb936a into 597dbc88ef704d0a8a736e149f091dd4(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:16:40,996 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e02d2d095200ac1f1a0b05c28cb936a: 2024-12-03T17:16:40,996 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., storeName=4e02d2d095200ac1f1a0b05c28cb936a/A, priority=13, startTime=1733246200821; duration=0sec 2024-12-03T17:16:40,996 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:40,996 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e02d2d095200ac1f1a0b05c28cb936a:A 2024-12-03T17:16:41,001 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e02d2d095200ac1f1a0b05c28cb936a#C#compaction#132 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:16:41,001 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/cc0a586c9219430ea0ee1d1cd7bf44c8 is 50, key is test_row_0/C:col10/1733246199196/Put/seqid=0 2024-12-03T17:16:41,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741988_1164 (size=12454) 2024-12-03T17:16:41,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:41,012 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203b6c853f3801749b6b08d973d3b6cb932_4e02d2d095200ac1f1a0b05c28cb936a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b6c853f3801749b6b08d973d3b6cb932_4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:41,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/e7347c4da1a44d829f28ef2691faa490, store: [table=TestAcidGuarantees family=A region=4e02d2d095200ac1f1a0b05c28cb936a] 2024-12-03T17:16:41,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/e7347c4da1a44d829f28ef2691faa490 is 175, key is test_row_0/A:col10/1733246200009/Put/seqid=0 2024-12-03T17:16:41,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741989_1165 (size=12847) 2024-12-03T17:16:41,053 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/cc0a586c9219430ea0ee1d1cd7bf44c8 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/cc0a586c9219430ea0ee1d1cd7bf44c8 2024-12-03T17:16:41,066 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4e02d2d095200ac1f1a0b05c28cb936a/C of 4e02d2d095200ac1f1a0b05c28cb936a into cc0a586c9219430ea0ee1d1cd7bf44c8(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:16:41,066 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e02d2d095200ac1f1a0b05c28cb936a: 2024-12-03T17:16:41,066 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., storeName=4e02d2d095200ac1f1a0b05c28cb936a/C, priority=13, startTime=1733246200823; duration=0sec 2024-12-03T17:16:41,066 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:41,066 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e02d2d095200ac1f1a0b05c28cb936a:C 2024-12-03T17:16:41,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-03T17:16:41,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741990_1166 (size=31255) 2024-12-03T17:16:41,083 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=312, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/e7347c4da1a44d829f28ef2691faa490 2024-12-03T17:16:41,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/fbb21d6d657146189abc4ef8ef33aa57 is 50, key is test_row_0/B:col10/1733246200009/Put/seqid=0 2024-12-03T17:16:41,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741991_1167 (size=12301) 2024-12-03T17:16:41,150 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/fbb21d6d657146189abc4ef8ef33aa57 2024-12-03T17:16:41,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. as already flushing 2024-12-03T17:16:41,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:41,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/e56df8016f624ba0aeaf53dbb832333b is 50, key is test_row_0/C:col10/1733246200009/Put/seqid=0 2024-12-03T17:16:41,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:41,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246261221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:41,228 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:41,228 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:41,228 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:41,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:41,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246261230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:41,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:41,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246261230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:41,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:41,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:41,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:41,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:41,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:41,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:41,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741992_1168 (size=12301) 2024-12-03T17:16:41,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:41,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246261245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:41,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:41,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:41,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:41,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:41,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246261334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:41,345 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:41,345 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:41,345 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:41,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:41,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246261354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:41,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:41,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:41,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:41,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:41,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246261354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:41,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:41,363 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:41,363 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:41,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:41,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246261367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:41,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:41,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:41,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:41,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:41,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246261555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:41,558 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:41,558 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:41,558 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:41,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:41,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246261573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:41,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:41,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246261573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:41,578 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:41,578 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:41,578 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:41,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:41,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:41,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:41,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-03T17:16:41,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:41,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246261594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:41,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:41,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:41,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:41,657 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.09 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/e56df8016f624ba0aeaf53dbb832333b 2024-12-03T17:16:41,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/e7347c4da1a44d829f28ef2691faa490 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/e7347c4da1a44d829f28ef2691faa490 2024-12-03T17:16:41,700 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/e7347c4da1a44d829f28ef2691faa490, entries=150, sequenceid=312, filesize=30.5 K 2024-12-03T17:16:41,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/fbb21d6d657146189abc4ef8ef33aa57 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/fbb21d6d657146189abc4ef8ef33aa57 2024-12-03T17:16:41,737 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/fbb21d6d657146189abc4ef8ef33aa57, entries=150, sequenceid=312, filesize=12.0 K 2024-12-03T17:16:41,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/e56df8016f624ba0aeaf53dbb832333b as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/e56df8016f624ba0aeaf53dbb832333b 2024-12-03T17:16:41,765 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/e56df8016f624ba0aeaf53dbb832333b, entries=150, sequenceid=312, filesize=12.0 K 2024-12-03T17:16:41,769 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~127.08 KB/130125, heapSize ~364.10 KB/372840, currentSize=67.09 KB/68700 for 4e02d2d095200ac1f1a0b05c28cb936a in 822ms, sequenceid=312, compaction requested=false 2024-12-03T17:16:41,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 4e02d2d095200ac1f1a0b05c28cb936a: 2024-12-03T17:16:41,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:41,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-12-03T17:16:41,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-12-03T17:16:41,781 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-12-03T17:16:41,781 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3290 sec 2024-12-03T17:16:41,786 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.3570 sec 2024-12-03T17:16:41,878 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4e02d2d095200ac1f1a0b05c28cb936a 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-03T17:16:41,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e02d2d095200ac1f1a0b05c28cb936a, store=A 2024-12-03T17:16:41,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:41,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e02d2d095200ac1f1a0b05c28cb936a, store=B 2024-12-03T17:16:41,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:41,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e02d2d095200ac1f1a0b05c28cb936a, store=C 2024-12-03T17:16:41,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:41,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:41,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412036009787cfb8a4e6bafa12d6b9fc52db3_4e02d2d095200ac1f1a0b05c28cb936a is 50, key is test_row_0/A:col10/1733246201224/Put/seqid=0 2024-12-03T17:16:41,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741993_1169 (size=14994) 2024-12-03T17:16:41,947 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:41,953 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412036009787cfb8a4e6bafa12d6b9fc52db3_4e02d2d095200ac1f1a0b05c28cb936a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036009787cfb8a4e6bafa12d6b9fc52db3_4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:41,958 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/eaaef6582ad44f8a9d3be40214b82914, store: [table=TestAcidGuarantees family=A region=4e02d2d095200ac1f1a0b05c28cb936a] 2024-12-03T17:16:41,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/eaaef6582ad44f8a9d3be40214b82914 is 175, key is test_row_0/A:col10/1733246201224/Put/seqid=0 2024-12-03T17:16:41,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:41,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246261963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:41,974 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:41,975 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:41,975 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:41,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:41,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246261971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:41,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:41,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246261972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:41,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:41,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246261974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:41,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:41,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:41,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:41,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:41,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:41,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:41,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:41,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:41,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:42,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741994_1170 (size=39949) 2024-12-03T17:16:42,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:42,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246262087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:42,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:42,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:42,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:42,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:42,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246262099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:42,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:42,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246262105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:42,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:42,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:42,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:42,107 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:42,107 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:42,107 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:42,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:42,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246262105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:42,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:42,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:42,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:42,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:42,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246262314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:42,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:42,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246262315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:42,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:42,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246262315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:42,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:42,333 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:42,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:42,333 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:42,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:42,333 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:42,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:42,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246262315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:42,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:42,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:42,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:42,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:42,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:42,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:42,408 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=329, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/eaaef6582ad44f8a9d3be40214b82914 2024-12-03T17:16:42,463 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/165444404af3442bad982d0aeb0be5c4 is 50, key is test_row_0/B:col10/1733246201224/Put/seqid=0 2024-12-03T17:16:42,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741995_1171 (size=12301) 2024-12-03T17:16:42,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-03T17:16:42,607 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:16:42,610 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T17:16:42,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-12-03T17:16:42,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-03T17:16:42,617 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T17:16:42,617 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T17:16:42,618 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T17:16:42,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:42,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246262644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:42,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:42,646 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:42,646 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:42,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246262644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:42,646 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:42,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:42,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246262645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:42,646 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:42,646 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:42,646 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:42,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:42,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:42,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:42,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:42,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246262645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:42,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:42,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:42,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:42,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-03T17:16:42,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-03T17:16:42,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:42,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. as already flushing 2024-12-03T17:16:42,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:42,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:42,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:42,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:42,924 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/165444404af3442bad982d0aeb0be5c4 2024-12-03T17:16:42,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-03T17:16:43,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/c563727546e7449aa63c12865a0c56f8 is 50, key is test_row_0/C:col10/1733246201224/Put/seqid=0 2024-12-03T17:16:43,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-03T17:16:43,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:43,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. as already flushing 2024-12-03T17:16:43,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:43,018 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:43,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:43,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:43,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741996_1172 (size=12301) 2024-12-03T17:16:43,063 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/c563727546e7449aa63c12865a0c56f8 2024-12-03T17:16:43,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/eaaef6582ad44f8a9d3be40214b82914 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/eaaef6582ad44f8a9d3be40214b82914 2024-12-03T17:16:43,096 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/eaaef6582ad44f8a9d3be40214b82914, entries=200, sequenceid=329, filesize=39.0 K 2024-12-03T17:16:43,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/165444404af3442bad982d0aeb0be5c4 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/165444404af3442bad982d0aeb0be5c4 2024-12-03T17:16:43,114 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/165444404af3442bad982d0aeb0be5c4, entries=150, sequenceid=329, filesize=12.0 K 2024-12-03T17:16:43,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/c563727546e7449aa63c12865a0c56f8 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/c563727546e7449aa63c12865a0c56f8 2024-12-03T17:16:43,135 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/c563727546e7449aa63c12865a0c56f8, entries=150, sequenceid=329, filesize=12.0 K 2024-12-03T17:16:43,138 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 4e02d2d095200ac1f1a0b05c28cb936a in 1260ms, sequenceid=329, compaction requested=true 2024-12-03T17:16:43,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4e02d2d095200ac1f1a0b05c28cb936a: 2024-12-03T17:16:43,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e02d2d095200ac1f1a0b05c28cb936a:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T17:16:43,138 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:16:43,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:43,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e02d2d095200ac1f1a0b05c28cb936a:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T17:16:43,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:43,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e02d2d095200ac1f1a0b05c28cb936a:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T17:16:43,138 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:16:43,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:16:43,140 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:16:43,140 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): 4e02d2d095200ac1f1a0b05c28cb936a/B is initiating minor compaction (all files) 2024-12-03T17:16:43,141 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e02d2d095200ac1f1a0b05c28cb936a/B in TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:43,141 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/e84ca4b239cb4f5e9ce06b2707a67900, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/fbb21d6d657146189abc4ef8ef33aa57, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/165444404af3442bad982d0aeb0be5c4] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp, totalSize=36.6 K 2024-12-03T17:16:43,141 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103005 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:16:43,141 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting e84ca4b239cb4f5e9ce06b2707a67900, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733246199196 2024-12-03T17:16:43,142 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): 4e02d2d095200ac1f1a0b05c28cb936a/A is initiating minor compaction (all files) 2024-12-03T17:16:43,142 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e02d2d095200ac1f1a0b05c28cb936a/A in TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:43,142 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/597dbc88ef704d0a8a736e149f091dd4, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/e7347c4da1a44d829f28ef2691faa490, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/eaaef6582ad44f8a9d3be40214b82914] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp, totalSize=100.6 K 2024-12-03T17:16:43,142 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:43,142 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. files: [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/597dbc88ef704d0a8a736e149f091dd4, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/e7347c4da1a44d829f28ef2691faa490, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/eaaef6582ad44f8a9d3be40214b82914] 2024-12-03T17:16:43,142 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting fbb21d6d657146189abc4ef8ef33aa57, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1733246200009 2024-12-03T17:16:43,143 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 597dbc88ef704d0a8a736e149f091dd4, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733246199196 2024-12-03T17:16:43,143 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 165444404af3442bad982d0aeb0be5c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1733246201200 2024-12-03T17:16:43,143 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7347c4da1a44d829f28ef2691faa490, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1733246200009 2024-12-03T17:16:43,144 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting eaaef6582ad44f8a9d3be40214b82914, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1733246201200 2024-12-03T17:16:43,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:43,163 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4e02d2d095200ac1f1a0b05c28cb936a 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T17:16:43,164 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4e02d2d095200ac1f1a0b05c28cb936a] 2024-12-03T17:16:43,165 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733246201958, max timestamp=1733246203161Num uniques 859; 2024-12-03T17:16:43,166 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T17:16:43,167 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733246201958, max timestamp=1733246203162Num uniques 859; 2024-12-03T17:16:43,167 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T17:16:43,168 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733246201958, max timestamp=1733246203161Num uniques 859; 2024-12-03T17:16:43,168 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T17:16:43,171 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e02d2d095200ac1f1a0b05c28cb936a#B#compaction#139 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:16:43,171 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/00a9fce678344d72967a44c7592efd03 is 50, key is test_row_0/B:col10/1733246201224/Put/seqid=0 2024-12-03T17:16:43,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e02d2d095200ac1f1a0b05c28cb936a, store=A 2024-12-03T17:16:43,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:43,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e02d2d095200ac1f1a0b05c28cb936a, store=B 2024-12-03T17:16:43,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:43,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e02d2d095200ac1f1a0b05c28cb936a, store=C 2024-12-03T17:16:43,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:43,175 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203ed46551afd024f79b6e8ea5fce8cb66f_4e02d2d095200ac1f1a0b05c28cb936a store=[table=TestAcidGuarantees family=A region=4e02d2d095200ac1f1a0b05c28cb936a] 2024-12-03T17:16:43,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-03T17:16:43,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:43,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. as already flushing 2024-12-03T17:16:43,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:43,184 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:43,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:43,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:43,187 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203ed46551afd024f79b6e8ea5fce8cb66f_4e02d2d095200ac1f1a0b05c28cb936a, store=[table=TestAcidGuarantees family=A region=4e02d2d095200ac1f1a0b05c28cb936a] 2024-12-03T17:16:43,188 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ed46551afd024f79b6e8ea5fce8cb66f_4e02d2d095200ac1f1a0b05c28cb936a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4e02d2d095200ac1f1a0b05c28cb936a] 2024-12-03T17:16:43,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ecee8cc24a49401db2c573316b296b81_4e02d2d095200ac1f1a0b05c28cb936a is 50, key is test_row_0/A:col10/1733246203160/Put/seqid=0 2024-12-03T17:16:43,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741997_1173 (size=12949) 2024-12-03T17:16:43,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741998_1174 (size=4469) 2024-12-03T17:16:43,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741999_1175 (size=12454) 2024-12-03T17:16:43,237 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:43,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-03T17:16:43,253 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ecee8cc24a49401db2c573316b296b81_4e02d2d095200ac1f1a0b05c28cb936a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ecee8cc24a49401db2c573316b296b81_4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:43,260 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/5f52b03aab6f43448af227058d4a7adb, store: [table=TestAcidGuarantees family=A region=4e02d2d095200ac1f1a0b05c28cb936a] 2024-12-03T17:16:43,260 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/5f52b03aab6f43448af227058d4a7adb is 175, key is test_row_0/A:col10/1733246203160/Put/seqid=0 2024-12-03T17:16:43,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742000_1176 (size=31255) 2024-12-03T17:16:43,312 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733246203278, max timestamp=1733246203307Num uniques 859; 2024-12-03T17:16:43,312 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T17:16:43,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733246203278, max timestamp=1733246203308Num uniques 859; 2024-12-03T17:16:43,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T17:16:43,316 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733246203278, max timestamp=1733246203308Num uniques 859; 2024-12-03T17:16:43,316 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T17:16:43,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-03T17:16:43,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:43,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. as already flushing 2024-12-03T17:16:43,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:43,347 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:43,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:43,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:43,398 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733246203319, max timestamp=1733246203383Num uniques 1118; 2024-12-03T17:16:43,398 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=9978, suffix heap size=144240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-12-03T17:16:43,403 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733246203319, max timestamp=1733246203383Num uniques 1118; 2024-12-03T17:16:43,403 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733246203319, max timestamp=1733246203383Num uniques 1118; 2024-12-03T17:16:43,403 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-12-03T17:16:43,403 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-12-03T17:16:43,505 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1733246203476, max timestamp=1733246203500Num uniques 1077; 2024-12-03T17:16:43,505 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=14972, suffix heap size=139976,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=327 2024-12-03T17:16:43,509 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1733246203476, max timestamp=1733246203500Num uniques 1027; 2024-12-03T17:16:43,509 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=14972, suffix heap size=134776,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=327 2024-12-03T17:16:43,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-03T17:16:43,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:43,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. as already flushing 2024-12-03T17:16:43,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:43,511 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:43,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:43,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:43,516 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1733246203476, max timestamp=1733246203500Num uniques 1027; 2024-12-03T17:16:43,516 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=14972, suffix heap size=134776,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=327 2024-12-03T17:16:43,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:43,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246263593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:43,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:43,612 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:43,612 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:43,612 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:43,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246263600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:43,616 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:43,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:43,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:43,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:43,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246263605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:43,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:43,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:43,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:43,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246263608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:43,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:43,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:43,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:43,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:43,629 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/00a9fce678344d72967a44c7592efd03 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/00a9fce678344d72967a44c7592efd03 2024-12-03T17:16:43,631 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e02d2d095200ac1f1a0b05c28cb936a#A#compaction#138 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:16:43,631 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/d188751342d34b3a8a2e9ba1b5e710a0 is 175, key is test_row_0/A:col10/1733246201224/Put/seqid=0 2024-12-03T17:16:43,660 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4e02d2d095200ac1f1a0b05c28cb936a/B of 4e02d2d095200ac1f1a0b05c28cb936a into 00a9fce678344d72967a44c7592efd03(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:16:43,660 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e02d2d095200ac1f1a0b05c28cb936a: 2024-12-03T17:16:43,660 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., storeName=4e02d2d095200ac1f1a0b05c28cb936a/B, priority=13, startTime=1733246203138; duration=0sec 2024-12-03T17:16:43,660 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:16:43,660 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e02d2d095200ac1f1a0b05c28cb936a:B 2024-12-03T17:16:43,660 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:16:43,662 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:16:43,663 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): 4e02d2d095200ac1f1a0b05c28cb936a/C is initiating minor compaction (all files) 2024-12-03T17:16:43,663 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e02d2d095200ac1f1a0b05c28cb936a/C in TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:43,663 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/cc0a586c9219430ea0ee1d1cd7bf44c8, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/e56df8016f624ba0aeaf53dbb832333b, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/c563727546e7449aa63c12865a0c56f8] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp, totalSize=36.6 K 2024-12-03T17:16:43,666 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting cc0a586c9219430ea0ee1d1cd7bf44c8, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733246199196 2024-12-03T17:16:43,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-03T17:16:43,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:43,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. as already flushing 2024-12-03T17:16:43,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:43,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:43,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:43,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742001_1177 (size=31903) 2024-12-03T17:16:43,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:43,678 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting e56df8016f624ba0aeaf53dbb832333b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1733246200009 2024-12-03T17:16:43,683 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting c563727546e7449aa63c12865a0c56f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1733246201200 2024-12-03T17:16:43,719 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=351, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/5f52b03aab6f43448af227058d4a7adb 2024-12-03T17:16:43,734 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e02d2d095200ac1f1a0b05c28cb936a#C#compaction#141 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:16:43,736 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/7fc76dfb4b744feba56efaaec13060b9 is 50, key is test_row_0/C:col10/1733246201224/Put/seqid=0 2024-12-03T17:16:43,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:43,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246263726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:43,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:43,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246263727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:43,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:43,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246263727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:43,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:43,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246263727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:43,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:43,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:43,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:43,743 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:43,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:43,743 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:43,743 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:43,743 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:43,743 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:43,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:43,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:43,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:43,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-03T17:16:43,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/7b1d04fdfd804a9691bcb01fa0c9c279 is 50, key is test_row_0/B:col10/1733246203160/Put/seqid=0 2024-12-03T17:16:43,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742002_1178 (size=12949) 2024-12-03T17:16:43,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-03T17:16:43,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:43,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. as already flushing 2024-12-03T17:16:43,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:43,840 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:43,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:43,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742003_1179 (size=12301) 2024-12-03T17:16:43,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:43,849 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/7b1d04fdfd804a9691bcb01fa0c9c279 2024-12-03T17:16:43,873 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/b970fbd799f64452ba86592af2f87c01 is 50, key is test_row_0/C:col10/1733246203160/Put/seqid=0 2024-12-03T17:16:43,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742004_1180 (size=12301) 2024-12-03T17:16:43,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:43,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246263957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:43,961 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:43,961 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:43,961 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:43,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:43,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246263958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:43,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:43,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246263958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:43,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:43,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246263959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:43,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:43,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:43,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:43,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:43,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:43,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:43,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:43,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:43,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:44,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-03T17:16:44,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:44,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. as already flushing 2024-12-03T17:16:44,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:44,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:44,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:44,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:44,087 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/d188751342d34b3a8a2e9ba1b5e710a0 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/d188751342d34b3a8a2e9ba1b5e710a0 2024-12-03T17:16:44,106 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4e02d2d095200ac1f1a0b05c28cb936a/A of 4e02d2d095200ac1f1a0b05c28cb936a into d188751342d34b3a8a2e9ba1b5e710a0(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:16:44,107 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e02d2d095200ac1f1a0b05c28cb936a: 2024-12-03T17:16:44,107 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., storeName=4e02d2d095200ac1f1a0b05c28cb936a/A, priority=13, startTime=1733246203138; duration=0sec 2024-12-03T17:16:44,107 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:44,107 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e02d2d095200ac1f1a0b05c28cb936a:A 2024-12-03T17:16:44,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-03T17:16:44,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:44,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. as already flushing 2024-12-03T17:16:44,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:44,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:44,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:44,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:44,256 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/7fc76dfb4b744feba56efaaec13060b9 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/7fc76dfb4b744feba56efaaec13060b9 2024-12-03T17:16:44,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:44,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246264266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:44,274 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:44,275 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:44,275 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:44,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:44,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246264274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:44,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:44,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246264274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:44,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:44,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:44,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:44,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:44,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:44,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:44,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:44,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246264274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:44,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:44,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:44,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:44,283 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4e02d2d095200ac1f1a0b05c28cb936a/C of 4e02d2d095200ac1f1a0b05c28cb936a into 7fc76dfb4b744feba56efaaec13060b9(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:16:44,283 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e02d2d095200ac1f1a0b05c28cb936a: 2024-12-03T17:16:44,283 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., storeName=4e02d2d095200ac1f1a0b05c28cb936a/C, priority=13, startTime=1733246203138; duration=0sec 2024-12-03T17:16:44,283 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:44,283 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e02d2d095200ac1f1a0b05c28cb936a:C 2024-12-03T17:16:44,319 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/b970fbd799f64452ba86592af2f87c01 2024-12-03T17:16:44,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/5f52b03aab6f43448af227058d4a7adb as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/5f52b03aab6f43448af227058d4a7adb 2024-12-03T17:16:44,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-03T17:16:44,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:44,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. as already flushing 2024-12-03T17:16:44,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:44,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:44,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:44,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:44,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/5f52b03aab6f43448af227058d4a7adb, entries=150, sequenceid=351, filesize=30.5 K 2024-12-03T17:16:44,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/7b1d04fdfd804a9691bcb01fa0c9c279 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/7b1d04fdfd804a9691bcb01fa0c9c279 2024-12-03T17:16:44,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/7b1d04fdfd804a9691bcb01fa0c9c279, entries=150, sequenceid=351, filesize=12.0 K 2024-12-03T17:16:44,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/b970fbd799f64452ba86592af2f87c01 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/b970fbd799f64452ba86592af2f87c01 2024-12-03T17:16:44,353 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/b970fbd799f64452ba86592af2f87c01, entries=150, sequenceid=351, filesize=12.0 K 2024-12-03T17:16:44,354 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~35.78 KB/36640, heapSize ~91.19 KB/93376, currentSize=154.31 KB/158010 for 4e02d2d095200ac1f1a0b05c28cb936a in 1192ms, sequenceid=351, compaction requested=false 2024-12-03T17:16:44,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4e02d2d095200ac1f1a0b05c28cb936a: 2024-12-03T17:16:44,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-03T17:16:44,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:44,489 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 4e02d2d095200ac1f1a0b05c28cb936a 3/3 column families, dataSize=154.31 KB heapSize=429 KB 2024-12-03T17:16:44,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e02d2d095200ac1f1a0b05c28cb936a, store=A 2024-12-03T17:16:44,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:44,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e02d2d095200ac1f1a0b05c28cb936a, store=B 2024-12-03T17:16:44,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:44,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e02d2d095200ac1f1a0b05c28cb936a, store=C 2024-12-03T17:16:44,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:44,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412032983d14bd5a54a59854e5c6edd41f909_4e02d2d095200ac1f1a0b05c28cb936a is 50, key is test_row_0/A:col10/1733246203605/Put/seqid=0 2024-12-03T17:16:44,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742005_1181 (size=12454) 2024-12-03T17:16:44,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-03T17:16:44,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:44,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. as already flushing 2024-12-03T17:16:44,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:44,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246264804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:44,805 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:44,805 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:44,805 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:44,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:44,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246264804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:44,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:44,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:44,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:44,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:44,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246264805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:44,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:44,813 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:44,813 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:44,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246264805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:44,813 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:44,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:44,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:44,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:44,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:44,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246264914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:44,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:44,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246264914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:44,930 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:44,930 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:44,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:44,930 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:44,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:44,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246264924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:44,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:44,931 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:44,932 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:44,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246264927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:44,932 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:44,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:44,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:44,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:44,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:44,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:44,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:44,947 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412032983d14bd5a54a59854e5c6edd41f909_4e02d2d095200ac1f1a0b05c28cb936a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412032983d14bd5a54a59854e5c6edd41f909_4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:44,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/913ef32ca498439d8b7910e0706e4d57, store: [table=TestAcidGuarantees family=A region=4e02d2d095200ac1f1a0b05c28cb936a] 2024-12-03T17:16:44,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/913ef32ca498439d8b7910e0706e4d57 is 175, key is test_row_0/A:col10/1733246203605/Put/seqid=0 2024-12-03T17:16:45,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742006_1182 (size=31255) 2024-12-03T17:16:45,031 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=425, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/913ef32ca498439d8b7910e0706e4d57 2024-12-03T17:16:45,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/0a659c8d00f14674b66142996adba76e is 50, key is test_row_0/B:col10/1733246203605/Put/seqid=0 2024-12-03T17:16:45,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742007_1183 (size=12301) 2024-12-03T17:16:45,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:45,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:45,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246265133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:45,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246265134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:45,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:45,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:45,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:45,142 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:45,142 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:45,142 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:45,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:45,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246265144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:45,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:45,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246265144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:45,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:45,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:45,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:45,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:45,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:45,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:45,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:45,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246265444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:45,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:45,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:45,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:45,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:45,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246265453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:45,457 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:45,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:45,457 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:45,458 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:45,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246265453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:45,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:45,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:45,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:45,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:45,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246265453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:45,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:45,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:45,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:45,512 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/0a659c8d00f14674b66142996adba76e 2024-12-03T17:16:45,533 INFO [Thread-531 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-531. 2024-12-03T17:16:45,533 DEBUG [Thread-531 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:16:45,533 DEBUG [Thread-531 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:45,533 DEBUG [Thread-531 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:45,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/d3de71bb8cdf44e29cc5d1d06eb06294 is 50, key is test_row_0/C:col10/1733246203605/Put/seqid=0 2024-12-03T17:16:45,534 INFO [Thread-529 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-529. 2024-12-03T17:16:45,534 DEBUG [Thread-529 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:16:45,534 DEBUG [Thread-529 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:45,534 DEBUG [Thread-529 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:45,534 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:16:45,534 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:16:45,539 INFO [Thread-527 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-527. 2024-12-03T17:16:45,539 DEBUG [Thread-527 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:16:45,539 DEBUG [Thread-527 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:45,539 DEBUG [Thread-527 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:45,539 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:16:45,540 INFO [Thread-533 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-533. 2024-12-03T17:16:45,540 DEBUG [Thread-533 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:16:45,540 DEBUG [Thread-533 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:45,540 DEBUG [Thread-533 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:45,540 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:16:45,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742008_1184 (size=12301) 2024-12-03T17:16:45,946 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/d3de71bb8cdf44e29cc5d1d06eb06294 2024-12-03T17:16:45,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/913ef32ca498439d8b7910e0706e4d57 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/913ef32ca498439d8b7910e0706e4d57 2024-12-03T17:16:45,959 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/913ef32ca498439d8b7910e0706e4d57, entries=150, sequenceid=425, filesize=30.5 K 2024-12-03T17:16:45,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/0a659c8d00f14674b66142996adba76e as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/0a659c8d00f14674b66142996adba76e 2024-12-03T17:16:45,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:45,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:45,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:45,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38960 deadline: 1733246265964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:45,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38968 deadline: 1733246265964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:45,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39032 deadline: 1733246265964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:45,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:45,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:39010 deadline: 1733246265964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:45,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:45,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:45,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:45,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:45,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:45,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:45,965 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:45,965 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:45,965 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:45,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:45,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:45,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:16:45,967 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/0a659c8d00f14674b66142996adba76e, entries=150, sequenceid=425, filesize=12.0 K 2024-12-03T17:16:45,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/d3de71bb8cdf44e29cc5d1d06eb06294 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/d3de71bb8cdf44e29cc5d1d06eb06294 2024-12-03T17:16:45,976 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/d3de71bb8cdf44e29cc5d1d06eb06294, entries=150, sequenceid=425, filesize=12.0 K 2024-12-03T17:16:45,977 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~428.95 KB/439248, currentSize=40.25 KB/41220 for 4e02d2d095200ac1f1a0b05c28cb936a in 1488ms, sequenceid=425, compaction requested=true 2024-12-03T17:16:45,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 4e02d2d095200ac1f1a0b05c28cb936a: 2024-12-03T17:16:45,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:45,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-12-03T17:16:45,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-12-03T17:16:45,982 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-12-03T17:16:45,982 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3610 sec 2024-12-03T17:16:45,984 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 3.3720 sec 2024-12-03T17:16:46,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:46,604 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4e02d2d095200ac1f1a0b05c28cb936a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T17:16:46,605 INFO [Thread-516 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-516. 2024-12-03T17:16:46,605 DEBUG [Thread-516 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:16:46,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e02d2d095200ac1f1a0b05c28cb936a, store=A 2024-12-03T17:16:46,605 DEBUG [Thread-516 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:46,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:46,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e02d2d095200ac1f1a0b05c28cb936a, store=B 2024-12-03T17:16:46,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:46,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e02d2d095200ac1f1a0b05c28cb936a, store=C 2024-12-03T17:16:46,605 DEBUG [Thread-516 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:46,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:46,605 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:16:46,611 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412034638720816c34efc8d462e498efb9fbd_4e02d2d095200ac1f1a0b05c28cb936a is 50, key is test_row_0/A:col10/1733246206603/Put/seqid=0 2024-12-03T17:16:46,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742009_1185 (size=12454) 2024-12-03T17:16:46,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-03T17:16:46,774 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:16:46,976 INFO [Thread-524 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-524. 2024-12-03T17:16:46,976 DEBUG [Thread-524 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:16:46,976 DEBUG [Thread-524 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:46,976 DEBUG [Thread-524 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:46,976 INFO [Thread-518 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-518. 2024-12-03T17:16:46,976 DEBUG [Thread-518 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:16:46,976 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:16:46,976 DEBUG [Thread-518 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:46,976 DEBUG [Thread-518 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:46,976 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:16:46,986 INFO [Thread-520 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-520. 2024-12-03T17:16:46,986 INFO [Thread-522 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-522. 2024-12-03T17:16:46,986 DEBUG [Thread-520 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:16:46,986 DEBUG [Thread-522 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:16:46,986 DEBUG [Thread-520 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:46,986 DEBUG [Thread-522 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:46,986 DEBUG [Thread-522 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:46,986 DEBUG [Thread-520 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:46,986 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:16:46,987 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:16:46,987 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-03T17:16:46,987 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-12-03T17:16:46,987 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 90 2024-12-03T17:16:46,987 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-12-03T17:16:46,987 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-12-03T17:16:46,987 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-12-03T17:16:46,987 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-03T17:16:46,987 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1373 2024-12-03T17:16:46,987 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1458 2024-12-03T17:16:46,987 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-03T17:16:46,987 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 665 2024-12-03T17:16:46,987 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1995 rows 2024-12-03T17:16:46,987 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 669 2024-12-03T17:16:46,987 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2007 rows 2024-12-03T17:16:46,987 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T17:16:46,987 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T17:16:46,987 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:46,987 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:46,988 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:16:46,988 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-03T17:16:46,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-03T17:16:46,992 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733246206992"}]},"ts":"1733246206992"} 2024-12-03T17:16:46,994 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-03T17:16:46,994 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-03T17:16:46,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-03T17:16:46,995 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T17:16:46,997 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4e02d2d095200ac1f1a0b05c28cb936a, UNASSIGN}] 2024-12-03T17:16:46,998 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4e02d2d095200ac1f1a0b05c28cb936a, UNASSIGN 2024-12-03T17:16:47,000 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=50 updating hbase:meta row=4e02d2d095200ac1f1a0b05c28cb936a, regionState=CLOSING, regionLocation=27e0f852a164,41167,1733246151593 2024-12-03T17:16:47,001 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4e02d2d095200ac1f1a0b05c28cb936a, UNASSIGN because future has completed 2024-12-03T17:16:47,002 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T17:16:47,002 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593}] 2024-12-03T17:16:47,016 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:47,033 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412034638720816c34efc8d462e498efb9fbd_4e02d2d095200ac1f1a0b05c28cb936a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412034638720816c34efc8d462e498efb9fbd_4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:47,039 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/6b5e695c8af34b5fb692c2ad48012be6, store: [table=TestAcidGuarantees family=A region=4e02d2d095200ac1f1a0b05c28cb936a] 2024-12-03T17:16:47,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/6b5e695c8af34b5fb692c2ad48012be6 is 175, key is test_row_0/A:col10/1733246206603/Put/seqid=0 2024-12-03T17:16:47,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742010_1186 (size=31255) 2024-12-03T17:16:47,047 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=435, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/6b5e695c8af34b5fb692c2ad48012be6 2024-12-03T17:16:47,061 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/3ee273f4b1f442a492386f03ca5755e8 is 50, key is test_row_0/B:col10/1733246206603/Put/seqid=0 2024-12-03T17:16:47,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742011_1187 (size=12301) 2024-12-03T17:16:47,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-03T17:16:47,155 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] handler.UnassignRegionHandler(122): Close 4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:47,155 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T17:16:47,156 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1722): Closing 4e02d2d095200ac1f1a0b05c28cb936a, disabling compactions & flushes 2024-12-03T17:16:47,156 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:47,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-03T17:16:47,472 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/3ee273f4b1f442a492386f03ca5755e8 2024-12-03T17:16:47,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/07ed1362c0664b9ca17f84fdf78653e1 is 50, key is test_row_0/C:col10/1733246206603/Put/seqid=0 2024-12-03T17:16:47,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742012_1188 (size=12301) 2024-12-03T17:16:47,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-03T17:16:47,884 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/07ed1362c0664b9ca17f84fdf78653e1 2024-12-03T17:16:47,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/6b5e695c8af34b5fb692c2ad48012be6 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/6b5e695c8af34b5fb692c2ad48012be6 2024-12-03T17:16:47,895 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/6b5e695c8af34b5fb692c2ad48012be6, entries=150, sequenceid=435, filesize=30.5 K 2024-12-03T17:16:47,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/3ee273f4b1f442a492386f03ca5755e8 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/3ee273f4b1f442a492386f03ca5755e8 2024-12-03T17:16:47,902 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/3ee273f4b1f442a492386f03ca5755e8, entries=150, sequenceid=435, filesize=12.0 K 2024-12-03T17:16:47,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/07ed1362c0664b9ca17f84fdf78653e1 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/07ed1362c0664b9ca17f84fdf78653e1 2024-12-03T17:16:47,909 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/07ed1362c0664b9ca17f84fdf78653e1, entries=150, sequenceid=435, filesize=12.0 K 2024-12-03T17:16:47,910 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=26.84 KB/27480 for 4e02d2d095200ac1f1a0b05c28cb936a in 1306ms, sequenceid=435, compaction requested=true 2024-12-03T17:16:47,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4e02d2d095200ac1f1a0b05c28cb936a: 2024-12-03T17:16:47,910 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:47,910 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:47,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e02d2d095200ac1f1a0b05c28cb936a:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T17:16:47,910 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. after waiting 0 ms 2024-12-03T17:16:47,910 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:47,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:47,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e02d2d095200ac1f1a0b05c28cb936a:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T17:16:47,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:47,910 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(2902): Flushing 4e02d2d095200ac1f1a0b05c28cb936a 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-03T17:16:47,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e02d2d095200ac1f1a0b05c28cb936a:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T17:16:47,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:16:47,910 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. because compaction request was cancelled 2024-12-03T17:16:47,910 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. because compaction request was cancelled 2024-12-03T17:16:47,910 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e02d2d095200ac1f1a0b05c28cb936a:A 2024-12-03T17:16:47,910 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e02d2d095200ac1f1a0b05c28cb936a:B 2024-12-03T17:16:47,910 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. because compaction request was cancelled 2024-12-03T17:16:47,910 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e02d2d095200ac1f1a0b05c28cb936a:C 2024-12-03T17:16:47,910 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e02d2d095200ac1f1a0b05c28cb936a, store=A 2024-12-03T17:16:47,910 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:47,910 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e02d2d095200ac1f1a0b05c28cb936a, store=B 2024-12-03T17:16:47,910 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:47,910 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e02d2d095200ac1f1a0b05c28cb936a, store=C 2024-12-03T17:16:47,911 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:47,917 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412033680448deb3f408d8fbe18bda9a40b74_4e02d2d095200ac1f1a0b05c28cb936a is 50, key is test_row_1/A:col10/1733246206984/Put/seqid=0 2024-12-03T17:16:47,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742013_1189 (size=9914) 2024-12-03T17:16:48,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-03T17:16:48,323 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:48,328 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412033680448deb3f408d8fbe18bda9a40b74_4e02d2d095200ac1f1a0b05c28cb936a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412033680448deb3f408d8fbe18bda9a40b74_4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:48,329 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/d6ba6fa7b4f74887a2ba549957bc9db0, store: [table=TestAcidGuarantees family=A region=4e02d2d095200ac1f1a0b05c28cb936a] 2024-12-03T17:16:48,330 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/d6ba6fa7b4f74887a2ba549957bc9db0 is 175, key is test_row_1/A:col10/1733246206984/Put/seqid=0 2024-12-03T17:16:48,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742014_1190 (size=22561) 2024-12-03T17:16:48,738 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=442, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/d6ba6fa7b4f74887a2ba549957bc9db0 2024-12-03T17:16:48,747 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/8a69dac7b0824a379b84384ed7be6abc is 50, key is test_row_1/B:col10/1733246206984/Put/seqid=0 2024-12-03T17:16:48,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742015_1191 (size=9857) 2024-12-03T17:16:49,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-03T17:16:49,152 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/8a69dac7b0824a379b84384ed7be6abc 2024-12-03T17:16:49,159 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/4c2bd0e6b1384babbf8bfbfc6fac4ebc is 50, key is test_row_1/C:col10/1733246206984/Put/seqid=0 2024-12-03T17:16:49,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742016_1192 (size=9857) 2024-12-03T17:16:49,399 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T17:16:49,567 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/4c2bd0e6b1384babbf8bfbfc6fac4ebc 2024-12-03T17:16:49,578 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/A/d6ba6fa7b4f74887a2ba549957bc9db0 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/d6ba6fa7b4f74887a2ba549957bc9db0 2024-12-03T17:16:49,586 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/d6ba6fa7b4f74887a2ba549957bc9db0, entries=100, sequenceid=442, filesize=22.0 K 2024-12-03T17:16:49,587 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/B/8a69dac7b0824a379b84384ed7be6abc as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/8a69dac7b0824a379b84384ed7be6abc 2024-12-03T17:16:49,592 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/8a69dac7b0824a379b84384ed7be6abc, entries=100, sequenceid=442, filesize=9.6 K 2024-12-03T17:16:49,593 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/.tmp/C/4c2bd0e6b1384babbf8bfbfc6fac4ebc as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/4c2bd0e6b1384babbf8bfbfc6fac4ebc 2024-12-03T17:16:49,599 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/4c2bd0e6b1384babbf8bfbfc6fac4ebc, entries=100, sequenceid=442, filesize=9.6 K 2024-12-03T17:16:49,600 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 4e02d2d095200ac1f1a0b05c28cb936a in 1690ms, sequenceid=442, compaction requested=true 2024-12-03T17:16:49,601 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/2c9b653936e64816a52127851c123cb6, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/bae8301b231245ea8c7173db21977730, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/8739b0eebdc64c60baf13f987ce3dd11, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/b11c602c8af84ebb8b7bef56c24f0975, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/798c3d2a8f0c4c36a06ca3863336ac87, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/019d440542c54f45b3d09dac09a5c53a, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/0c678eddcf3c43d2a5fe449edfb6426f, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/29b6adb605054138af14279dc3c65489, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/940c0724f2d749c1beec268e057a641e, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/2f5a1b9d577f400d853eb283a3b297a0, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/77748662e1f84faeae78033999be84e0, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/1ac542e15ae0463d85f3f90192e4c444, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/e72595ba7f88482e8be46fed73e77854, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/5a479282410143cba911c6d295dbe23f, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/3391385ac8c040deb36c4c56ab6b68dd, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/597dbc88ef704d0a8a736e149f091dd4, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/13543fc2a6454c5481e80581e87e091d, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/e7347c4da1a44d829f28ef2691faa490, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/eaaef6582ad44f8a9d3be40214b82914] to archive 2024-12-03T17:16:49,602 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T17:16:49,604 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/2c9b653936e64816a52127851c123cb6 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/2c9b653936e64816a52127851c123cb6 2024-12-03T17:16:49,605 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/bae8301b231245ea8c7173db21977730 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/bae8301b231245ea8c7173db21977730 2024-12-03T17:16:49,607 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/8739b0eebdc64c60baf13f987ce3dd11 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/8739b0eebdc64c60baf13f987ce3dd11 2024-12-03T17:16:49,608 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/b11c602c8af84ebb8b7bef56c24f0975 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/b11c602c8af84ebb8b7bef56c24f0975 2024-12-03T17:16:49,609 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/798c3d2a8f0c4c36a06ca3863336ac87 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/798c3d2a8f0c4c36a06ca3863336ac87 2024-12-03T17:16:49,611 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/019d440542c54f45b3d09dac09a5c53a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/019d440542c54f45b3d09dac09a5c53a 2024-12-03T17:16:49,612 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/0c678eddcf3c43d2a5fe449edfb6426f to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/0c678eddcf3c43d2a5fe449edfb6426f 2024-12-03T17:16:49,613 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/29b6adb605054138af14279dc3c65489 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/29b6adb605054138af14279dc3c65489 2024-12-03T17:16:49,614 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/940c0724f2d749c1beec268e057a641e to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/940c0724f2d749c1beec268e057a641e 2024-12-03T17:16:49,615 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/2f5a1b9d577f400d853eb283a3b297a0 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/2f5a1b9d577f400d853eb283a3b297a0 2024-12-03T17:16:49,616 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/77748662e1f84faeae78033999be84e0 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/77748662e1f84faeae78033999be84e0 2024-12-03T17:16:49,617 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/1ac542e15ae0463d85f3f90192e4c444 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/1ac542e15ae0463d85f3f90192e4c444 2024-12-03T17:16:49,618 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/e72595ba7f88482e8be46fed73e77854 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/e72595ba7f88482e8be46fed73e77854 2024-12-03T17:16:49,619 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/5a479282410143cba911c6d295dbe23f to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/5a479282410143cba911c6d295dbe23f 2024-12-03T17:16:49,620 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/3391385ac8c040deb36c4c56ab6b68dd to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/3391385ac8c040deb36c4c56ab6b68dd 2024-12-03T17:16:49,621 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/597dbc88ef704d0a8a736e149f091dd4 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/597dbc88ef704d0a8a736e149f091dd4 2024-12-03T17:16:49,622 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/13543fc2a6454c5481e80581e87e091d to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/13543fc2a6454c5481e80581e87e091d 2024-12-03T17:16:49,624 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/e7347c4da1a44d829f28ef2691faa490 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/e7347c4da1a44d829f28ef2691faa490 2024-12-03T17:16:49,625 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/eaaef6582ad44f8a9d3be40214b82914 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/eaaef6582ad44f8a9d3be40214b82914 2024-12-03T17:16:49,627 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/e6f8efa8a5e242a69df9959922fb65d1, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/2c3761f2c37443249a553c32a0b9c895, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/8e3613a780b24a62ae00311c53459bc4, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/ffb075ec355144cf904297a5949bd461, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/61793b763dba4d42bfb66476f935428c, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/dca990e2a0224cff9b6f43fd5d0e4ca8, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/12f4fe45cc4e4a7b997fe059a4f0b3cc, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/233f9d05676e4be3b928c2f4d0dfa852, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/c1b5ade639fe459f96e6bc619b700106, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/1e7a7eab40ca4a9d99e186587aa3f782, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/9f24770ba17f49a19f07a04cbd04e7c7, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/eb69f9610f36488c9f6facae35a458b5, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/fdb16c78fc9740d2b579c1ed33b60c6b, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/e268adbe098b4bfba6a8b7d49320107d, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/409beb3dd0c040ed914961b69a3d7341, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/e84ca4b239cb4f5e9ce06b2707a67900, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/e7e0b84ff17e438fad92e90b8e2607bb, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/fbb21d6d657146189abc4ef8ef33aa57, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/165444404af3442bad982d0aeb0be5c4] to archive 2024-12-03T17:16:49,628 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T17:16:49,630 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/e6f8efa8a5e242a69df9959922fb65d1 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/e6f8efa8a5e242a69df9959922fb65d1 2024-12-03T17:16:49,631 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/2c3761f2c37443249a553c32a0b9c895 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/2c3761f2c37443249a553c32a0b9c895 2024-12-03T17:16:49,632 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/8e3613a780b24a62ae00311c53459bc4 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/8e3613a780b24a62ae00311c53459bc4 2024-12-03T17:16:49,634 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/ffb075ec355144cf904297a5949bd461 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/ffb075ec355144cf904297a5949bd461 2024-12-03T17:16:49,635 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/61793b763dba4d42bfb66476f935428c to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/61793b763dba4d42bfb66476f935428c 2024-12-03T17:16:49,636 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/dca990e2a0224cff9b6f43fd5d0e4ca8 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/dca990e2a0224cff9b6f43fd5d0e4ca8 2024-12-03T17:16:49,637 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/12f4fe45cc4e4a7b997fe059a4f0b3cc to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/12f4fe45cc4e4a7b997fe059a4f0b3cc 2024-12-03T17:16:49,638 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/233f9d05676e4be3b928c2f4d0dfa852 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/233f9d05676e4be3b928c2f4d0dfa852 2024-12-03T17:16:49,640 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/c1b5ade639fe459f96e6bc619b700106 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/c1b5ade639fe459f96e6bc619b700106 2024-12-03T17:16:49,641 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/1e7a7eab40ca4a9d99e186587aa3f782 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/1e7a7eab40ca4a9d99e186587aa3f782 2024-12-03T17:16:49,642 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/9f24770ba17f49a19f07a04cbd04e7c7 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/9f24770ba17f49a19f07a04cbd04e7c7 2024-12-03T17:16:49,643 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/eb69f9610f36488c9f6facae35a458b5 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/eb69f9610f36488c9f6facae35a458b5 2024-12-03T17:16:49,644 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/fdb16c78fc9740d2b579c1ed33b60c6b to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/fdb16c78fc9740d2b579c1ed33b60c6b 2024-12-03T17:16:49,645 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/e268adbe098b4bfba6a8b7d49320107d to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/e268adbe098b4bfba6a8b7d49320107d 2024-12-03T17:16:49,646 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/409beb3dd0c040ed914961b69a3d7341 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/409beb3dd0c040ed914961b69a3d7341 2024-12-03T17:16:49,647 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/e84ca4b239cb4f5e9ce06b2707a67900 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/e84ca4b239cb4f5e9ce06b2707a67900 2024-12-03T17:16:49,648 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/e7e0b84ff17e438fad92e90b8e2607bb to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/e7e0b84ff17e438fad92e90b8e2607bb 2024-12-03T17:16:49,649 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/fbb21d6d657146189abc4ef8ef33aa57 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/fbb21d6d657146189abc4ef8ef33aa57 2024-12-03T17:16:49,651 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/165444404af3442bad982d0aeb0be5c4 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/165444404af3442bad982d0aeb0be5c4 2024-12-03T17:16:49,652 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/8911ef322c5f46bc83d81f738b59fcdc, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/9c246221e8f84e909b6ac1be23c0a4ba, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/e7b73b54d2924a0fa777fd318b512704, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/f0de18dc3a1b45d299aa119c531a1127, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/615be05b4f5345ec876cfc3805a0de45, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/0930543cebeb41f2af31f6b43422d954, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/2be9ea593b4245b38c7accc45dc49ba8, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/ab4c98384c744c17842ee49a685b492f, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/e6ed175c84124febb1391041954099e8, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/d72b9698989643eab4a1683ebe93a822, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/3824d94afd544feb9d9e54a4c94412f8, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/1d7a810fa8274728a8ab77e759814941, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/141c92d5224f43f49ee78788011d3d78, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/3de3e35a453a48918cfaf057c6b0a124, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/40c9ca9405954afe92fb4b34ce5737ea, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/cc0a586c9219430ea0ee1d1cd7bf44c8, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/44a45684da3841ce8b389d9a1bb25b57, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/e56df8016f624ba0aeaf53dbb832333b, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/c563727546e7449aa63c12865a0c56f8] to archive 2024-12-03T17:16:49,653 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T17:16:49,655 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/8911ef322c5f46bc83d81f738b59fcdc to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/8911ef322c5f46bc83d81f738b59fcdc 2024-12-03T17:16:49,656 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/9c246221e8f84e909b6ac1be23c0a4ba to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/9c246221e8f84e909b6ac1be23c0a4ba 2024-12-03T17:16:49,657 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/e7b73b54d2924a0fa777fd318b512704 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/e7b73b54d2924a0fa777fd318b512704 2024-12-03T17:16:49,658 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/f0de18dc3a1b45d299aa119c531a1127 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/f0de18dc3a1b45d299aa119c531a1127 2024-12-03T17:16:49,659 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/615be05b4f5345ec876cfc3805a0de45 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/615be05b4f5345ec876cfc3805a0de45 2024-12-03T17:16:49,660 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/0930543cebeb41f2af31f6b43422d954 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/0930543cebeb41f2af31f6b43422d954 2024-12-03T17:16:49,661 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/2be9ea593b4245b38c7accc45dc49ba8 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/2be9ea593b4245b38c7accc45dc49ba8 2024-12-03T17:16:49,662 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/ab4c98384c744c17842ee49a685b492f to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/ab4c98384c744c17842ee49a685b492f 2024-12-03T17:16:49,663 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/e6ed175c84124febb1391041954099e8 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/e6ed175c84124febb1391041954099e8 2024-12-03T17:16:49,664 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/d72b9698989643eab4a1683ebe93a822 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/d72b9698989643eab4a1683ebe93a822 2024-12-03T17:16:49,665 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/3824d94afd544feb9d9e54a4c94412f8 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/3824d94afd544feb9d9e54a4c94412f8 2024-12-03T17:16:49,666 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/1d7a810fa8274728a8ab77e759814941 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/1d7a810fa8274728a8ab77e759814941 2024-12-03T17:16:49,667 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/141c92d5224f43f49ee78788011d3d78 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/141c92d5224f43f49ee78788011d3d78 2024-12-03T17:16:49,668 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/3de3e35a453a48918cfaf057c6b0a124 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/3de3e35a453a48918cfaf057c6b0a124 2024-12-03T17:16:49,669 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/40c9ca9405954afe92fb4b34ce5737ea to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/40c9ca9405954afe92fb4b34ce5737ea 2024-12-03T17:16:49,670 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/cc0a586c9219430ea0ee1d1cd7bf44c8 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/cc0a586c9219430ea0ee1d1cd7bf44c8 2024-12-03T17:16:49,671 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/44a45684da3841ce8b389d9a1bb25b57 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/44a45684da3841ce8b389d9a1bb25b57 2024-12-03T17:16:49,672 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/e56df8016f624ba0aeaf53dbb832333b to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/e56df8016f624ba0aeaf53dbb832333b 2024-12-03T17:16:49,673 DEBUG [StoreCloser-TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/c563727546e7449aa63c12865a0c56f8 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/c563727546e7449aa63c12865a0c56f8 2024-12-03T17:16:49,677 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/recovered.edits/445.seqid, newMaxSeqId=445, maxSeqId=4 2024-12-03T17:16:49,678 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. 2024-12-03T17:16:49,678 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1676): Region close journal for 4e02d2d095200ac1f1a0b05c28cb936a: Waiting for close lock at 1733246207156Running coprocessor pre-close hooks at 1733246207156Disabling compacts and flushes for region at 1733246207156Disabling writes for close at 1733246207910 (+754 ms)Obtaining lock to block concurrent updates at 1733246207910Preparing flush snapshotting stores in 4e02d2d095200ac1f1a0b05c28cb936a at 1733246207910Finished memstore snapshotting TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733246207911 (+1 ms)Flushing stores of TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a. at 1733246207911Flushing 4e02d2d095200ac1f1a0b05c28cb936a/A: creating writer at 1733246207911Flushing mob file 4e02d2d095200ac1f1a0b05c28cb936a/A: appending metadata at 1733246207917 (+6 ms)Flushing mob file 4e02d2d095200ac1f1a0b05c28cb936a/A: closing flushed file at 1733246207917Flushing 4e02d2d095200ac1f1a0b05c28cb936a/A: appending metadata at 1733246208329 (+412 ms)Flushing 4e02d2d095200ac1f1a0b05c28cb936a/A: closing flushed file at 1733246208329Flushing 4e02d2d095200ac1f1a0b05c28cb936a/B: creating writer at 1733246208743 (+414 ms)Flushing 4e02d2d095200ac1f1a0b05c28cb936a/B: appending metadata at 1733246208746 (+3 ms)Flushing 4e02d2d095200ac1f1a0b05c28cb936a/B: closing flushed file at 1733246208747 (+1 ms)Flushing 4e02d2d095200ac1f1a0b05c28cb936a/C: creating writer at 1733246209156 (+409 ms)Flushing 4e02d2d095200ac1f1a0b05c28cb936a/C: appending metadata at 1733246209159 (+3 ms)Flushing 4e02d2d095200ac1f1a0b05c28cb936a/C: closing flushed file at 1733246209159Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d9eee21: reopening flushed file at 1733246209576 (+417 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a9aeaa2: reopening flushed file at 1733246209586 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@400857b4: reopening flushed file at 1733246209593 (+7 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 4e02d2d095200ac1f1a0b05c28cb936a in 1690ms, sequenceid=442, compaction requested=true at 1733246209600 (+7 ms)Writing region close event to WAL at 1733246209674 (+74 ms)Running coprocessor post-close hooks at 1733246209678 (+4 ms)Closed at 1733246209678 2024-12-03T17:16:49,680 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] handler.UnassignRegionHandler(157): Closed 4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:49,681 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=50 updating hbase:meta row=4e02d2d095200ac1f1a0b05c28cb936a, regionState=CLOSED 2024-12-03T17:16:49,682 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=50, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 because future has completed 2024-12-03T17:16:49,685 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-12-03T17:16:49,685 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; CloseRegionProcedure 4e02d2d095200ac1f1a0b05c28cb936a, server=27e0f852a164,41167,1733246151593 in 2.6810 sec 2024-12-03T17:16:49,687 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=49 2024-12-03T17:16:49,687 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=49, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4e02d2d095200ac1f1a0b05c28cb936a, UNASSIGN in 2.6880 sec 2024-12-03T17:16:49,690 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-12-03T17:16:49,690 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.6940 sec 2024-12-03T17:16:49,691 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733246209691"}]},"ts":"1733246209691"} 2024-12-03T17:16:49,693 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-03T17:16:49,693 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-03T17:16:49,695 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.7050 sec 2024-12-03T17:16:51,122 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-03T17:16:51,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-03T17:16:51,174 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:16:51,175 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-03T17:16:51,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T17:16:51,180 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=52, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T17:16:51,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-03T17:16:51,182 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=52, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T17:16:51,185 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:51,189 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A, FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B, FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C, FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/recovered.edits] 2024-12-03T17:16:51,193 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/5f52b03aab6f43448af227058d4a7adb to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/5f52b03aab6f43448af227058d4a7adb 2024-12-03T17:16:51,195 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/6b5e695c8af34b5fb692c2ad48012be6 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/6b5e695c8af34b5fb692c2ad48012be6 2024-12-03T17:16:51,197 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/913ef32ca498439d8b7910e0706e4d57 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/913ef32ca498439d8b7910e0706e4d57 2024-12-03T17:16:51,199 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/d188751342d34b3a8a2e9ba1b5e710a0 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/d188751342d34b3a8a2e9ba1b5e710a0 2024-12-03T17:16:51,201 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/d6ba6fa7b4f74887a2ba549957bc9db0 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/A/d6ba6fa7b4f74887a2ba549957bc9db0 2024-12-03T17:16:51,207 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/00a9fce678344d72967a44c7592efd03 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/00a9fce678344d72967a44c7592efd03 2024-12-03T17:16:51,210 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/0a659c8d00f14674b66142996adba76e to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/0a659c8d00f14674b66142996adba76e 2024-12-03T17:16:51,212 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/3ee273f4b1f442a492386f03ca5755e8 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/3ee273f4b1f442a492386f03ca5755e8 2024-12-03T17:16:51,217 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/7b1d04fdfd804a9691bcb01fa0c9c279 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/7b1d04fdfd804a9691bcb01fa0c9c279 2024-12-03T17:16:51,219 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/8a69dac7b0824a379b84384ed7be6abc to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/B/8a69dac7b0824a379b84384ed7be6abc 2024-12-03T17:16:51,223 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/07ed1362c0664b9ca17f84fdf78653e1 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/07ed1362c0664b9ca17f84fdf78653e1 2024-12-03T17:16:51,226 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/4c2bd0e6b1384babbf8bfbfc6fac4ebc to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/4c2bd0e6b1384babbf8bfbfc6fac4ebc 2024-12-03T17:16:51,228 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/7fc76dfb4b744feba56efaaec13060b9 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/7fc76dfb4b744feba56efaaec13060b9 2024-12-03T17:16:51,229 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/b970fbd799f64452ba86592af2f87c01 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/b970fbd799f64452ba86592af2f87c01 2024-12-03T17:16:51,233 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/d3de71bb8cdf44e29cc5d1d06eb06294 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/C/d3de71bb8cdf44e29cc5d1d06eb06294 2024-12-03T17:16:51,239 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/recovered.edits/445.seqid to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a/recovered.edits/445.seqid 2024-12-03T17:16:51,240 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:51,240 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-03T17:16:51,241 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-03T17:16:51,242 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-03T17:16:51,247 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412030bff014e774c4c43b46ce7d815875091_4e02d2d095200ac1f1a0b05c28cb936a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412030bff014e774c4c43b46ce7d815875091_4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:51,249 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412032983d14bd5a54a59854e5c6edd41f909_4e02d2d095200ac1f1a0b05c28cb936a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412032983d14bd5a54a59854e5c6edd41f909_4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:51,250 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412033680448deb3f408d8fbe18bda9a40b74_4e02d2d095200ac1f1a0b05c28cb936a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412033680448deb3f408d8fbe18bda9a40b74_4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:51,252 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412034638720816c34efc8d462e498efb9fbd_4e02d2d095200ac1f1a0b05c28cb936a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412034638720816c34efc8d462e498efb9fbd_4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:51,253 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412035ad294dcef824b2e80949859344f0026_4e02d2d095200ac1f1a0b05c28cb936a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412035ad294dcef824b2e80949859344f0026_4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:51,254 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036009787cfb8a4e6bafa12d6b9fc52db3_4e02d2d095200ac1f1a0b05c28cb936a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036009787cfb8a4e6bafa12d6b9fc52db3_4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:51,256 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036c57e851c5f34b928a2402b4fdbabf26_4e02d2d095200ac1f1a0b05c28cb936a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036c57e851c5f34b928a2402b4fdbabf26_4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:51,260 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412037cef23a585444199bc282af113c7738d_4e02d2d095200ac1f1a0b05c28cb936a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412037cef23a585444199bc282af113c7738d_4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:51,261 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412038a136ee16dc04e08a7318781802a6fa8_4e02d2d095200ac1f1a0b05c28cb936a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412038a136ee16dc04e08a7318781802a6fa8_4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:51,264 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039350a5c31d4b48a5a2bb569ec0428095_4e02d2d095200ac1f1a0b05c28cb936a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039350a5c31d4b48a5a2bb569ec0428095_4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:51,267 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203a08de2a084434707928e985d0f3aa704_4e02d2d095200ac1f1a0b05c28cb936a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203a08de2a084434707928e985d0f3aa704_4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:51,269 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b6c853f3801749b6b08d973d3b6cb932_4e02d2d095200ac1f1a0b05c28cb936a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b6c853f3801749b6b08d973d3b6cb932_4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:51,271 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b92e6767fc1d40648443c81528062097_4e02d2d095200ac1f1a0b05c28cb936a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203b92e6767fc1d40648443c81528062097_4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:51,274 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203e13194aa7b164ebf87655b899fda471a_4e02d2d095200ac1f1a0b05c28cb936a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203e13194aa7b164ebf87655b899fda471a_4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:51,276 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203e4876b489c174d88a4e74fc7147b9840_4e02d2d095200ac1f1a0b05c28cb936a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203e4876b489c174d88a4e74fc7147b9840_4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:51,278 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203e7520f740a034a10b100a57f4948670b_4e02d2d095200ac1f1a0b05c28cb936a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203e7520f740a034a10b100a57f4948670b_4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:51,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-03T17:16:51,284 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ecc1dd994939412db895a41798347a3b_4e02d2d095200ac1f1a0b05c28cb936a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ecc1dd994939412db895a41798347a3b_4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:51,289 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ecee8cc24a49401db2c573316b296b81_4e02d2d095200ac1f1a0b05c28cb936a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ecee8cc24a49401db2c573316b296b81_4e02d2d095200ac1f1a0b05c28cb936a 2024-12-03T17:16:51,290 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-03T17:16:51,293 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=52, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T17:16:51,297 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-03T17:16:51,302 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-03T17:16:51,305 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=52, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T17:16:51,305 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-03T17:16:51,305 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733246211305"}]},"ts":"9223372036854775807"} 2024-12-03T17:16:51,308 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-03T17:16:51,308 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 4e02d2d095200ac1f1a0b05c28cb936a, NAME => 'TestAcidGuarantees,,1733246183019.4e02d2d095200ac1f1a0b05c28cb936a.', STARTKEY => '', ENDKEY => ''}] 2024-12-03T17:16:51,309 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-03T17:16:51,309 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733246211309"}]},"ts":"9223372036854775807"} 2024-12-03T17:16:51,311 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-03T17:16:51,312 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=52, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T17:16:51,313 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 137 msec 2024-12-03T17:16:51,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-03T17:16:51,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-03T17:16:51,501 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:16:51,513 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=252 (was 249) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_782644355_22 at /127.0.0.1:52242 [Waiting for operation #293] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1405682164_22 at /127.0.0.1:38988 [Waiting for operation #403] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f784850d-ec8c-cbd4-e545-debb5ca850cc/cluster_481ba349-0fa7-2aa3-a4cc-d5a666163a05/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f784850d-ec8c-cbd4-e545-debb5ca850cc/cluster_481ba349-0fa7-2aa3-a4cc-d5a666163a05/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_782644355_22 at /127.0.0.1:49548 [Waiting for operation #535] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1405682164_22 at /127.0.0.1:52238 [Waiting for operation #294] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1527 (was 1483) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4563 (was 4726) 2024-12-03T17:16:51,528 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=252, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=1527, ProcessCount=11, AvailableMemoryMB=4560 2024-12-03T17:16:51,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T17:16:51,531 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T17:16:51,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=53, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-03T17:16:51,536 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T17:16:51,536 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:51,536 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 53 2024-12-03T17:16:51,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-12-03T17:16:51,538 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T17:16:51,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742017_1193 (size=1061) 2024-12-03T17:16:51,601 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4 2024-12-03T17:16:51,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-12-03T17:16:51,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742018_1194 (size=53) 2024-12-03T17:16:51,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-12-03T17:16:52,076 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T17:16:52,076 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing f5a2a3f1964bd8ecfa7908952eaec708, disabling compactions & flushes 2024-12-03T17:16:52,076 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:16:52,076 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:16:52,076 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. after waiting 0 ms 2024-12-03T17:16:52,076 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:16:52,076 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:16:52,076 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for f5a2a3f1964bd8ecfa7908952eaec708: Waiting for close lock at 1733246212076Disabling compacts and flushes for region at 1733246212076Disabling writes for close at 1733246212076Writing region close event to WAL at 1733246212076Closed at 1733246212076 2024-12-03T17:16:52,077 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T17:16:52,078 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733246212077"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733246212077"}]},"ts":"1733246212077"} 2024-12-03T17:16:52,079 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T17:16:52,081 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T17:16:52,081 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733246212081"}]},"ts":"1733246212081"} 2024-12-03T17:16:52,083 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-03T17:16:52,083 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f5a2a3f1964bd8ecfa7908952eaec708, ASSIGN}] 2024-12-03T17:16:52,085 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f5a2a3f1964bd8ecfa7908952eaec708, ASSIGN 2024-12-03T17:16:52,085 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f5a2a3f1964bd8ecfa7908952eaec708, ASSIGN; state=OFFLINE, location=27e0f852a164,41167,1733246151593; forceNewPlan=false, retain=false 2024-12-03T17:16:52,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-12-03T17:16:52,236 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=f5a2a3f1964bd8ecfa7908952eaec708, regionState=OPENING, regionLocation=27e0f852a164,41167,1733246151593 2024-12-03T17:16:52,238 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f5a2a3f1964bd8ecfa7908952eaec708, ASSIGN because future has completed 2024-12-03T17:16:52,239 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593}] 2024-12-03T17:16:52,395 INFO [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:16:52,395 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} 2024-12-03T17:16:52,395 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f5a2a3f1964bd8ecfa7908952eaec708 2024-12-03T17:16:52,395 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T17:16:52,395 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for f5a2a3f1964bd8ecfa7908952eaec708 2024-12-03T17:16:52,395 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for f5a2a3f1964bd8ecfa7908952eaec708 2024-12-03T17:16:52,397 INFO [StoreOpener-f5a2a3f1964bd8ecfa7908952eaec708-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f5a2a3f1964bd8ecfa7908952eaec708 2024-12-03T17:16:52,399 INFO [StoreOpener-f5a2a3f1964bd8ecfa7908952eaec708-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T17:16:52,399 INFO [StoreOpener-f5a2a3f1964bd8ecfa7908952eaec708-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f5a2a3f1964bd8ecfa7908952eaec708 columnFamilyName A 2024-12-03T17:16:52,399 DEBUG [StoreOpener-f5a2a3f1964bd8ecfa7908952eaec708-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:52,400 INFO [StoreOpener-f5a2a3f1964bd8ecfa7908952eaec708-1 {}] regionserver.HStore(327): Store=f5a2a3f1964bd8ecfa7908952eaec708/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T17:16:52,400 INFO [StoreOpener-f5a2a3f1964bd8ecfa7908952eaec708-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f5a2a3f1964bd8ecfa7908952eaec708 2024-12-03T17:16:52,401 INFO [StoreOpener-f5a2a3f1964bd8ecfa7908952eaec708-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T17:16:52,401 INFO [StoreOpener-f5a2a3f1964bd8ecfa7908952eaec708-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f5a2a3f1964bd8ecfa7908952eaec708 columnFamilyName B 2024-12-03T17:16:52,402 DEBUG [StoreOpener-f5a2a3f1964bd8ecfa7908952eaec708-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:52,402 INFO [StoreOpener-f5a2a3f1964bd8ecfa7908952eaec708-1 {}] regionserver.HStore(327): Store=f5a2a3f1964bd8ecfa7908952eaec708/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T17:16:52,402 INFO [StoreOpener-f5a2a3f1964bd8ecfa7908952eaec708-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f5a2a3f1964bd8ecfa7908952eaec708 2024-12-03T17:16:52,404 INFO [StoreOpener-f5a2a3f1964bd8ecfa7908952eaec708-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T17:16:52,404 INFO [StoreOpener-f5a2a3f1964bd8ecfa7908952eaec708-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f5a2a3f1964bd8ecfa7908952eaec708 columnFamilyName C 2024-12-03T17:16:52,404 DEBUG [StoreOpener-f5a2a3f1964bd8ecfa7908952eaec708-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:16:52,404 INFO [StoreOpener-f5a2a3f1964bd8ecfa7908952eaec708-1 {}] regionserver.HStore(327): Store=f5a2a3f1964bd8ecfa7908952eaec708/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T17:16:52,405 INFO [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:16:52,405 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for f5a2a3f1964bd8ecfa7908952eaec708 2024-12-03T17:16:52,405 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708 2024-12-03T17:16:52,406 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708 2024-12-03T17:16:52,407 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for f5a2a3f1964bd8ecfa7908952eaec708 2024-12-03T17:16:52,407 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for f5a2a3f1964bd8ecfa7908952eaec708 2024-12-03T17:16:52,408 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T17:16:52,409 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for f5a2a3f1964bd8ecfa7908952eaec708 2024-12-03T17:16:52,412 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T17:16:52,412 INFO [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened f5a2a3f1964bd8ecfa7908952eaec708; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67258144, jitterRate=0.002224445343017578}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T17:16:52,412 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f5a2a3f1964bd8ecfa7908952eaec708 2024-12-03T17:16:52,413 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for f5a2a3f1964bd8ecfa7908952eaec708: Running coprocessor pre-open hook at 1733246212395Writing region info on filesystem at 1733246212395Initializing all the Stores at 1733246212396 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733246212396Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733246212397 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733246212397Cleaning up temporary data from old regions at 1733246212407 (+10 ms)Running coprocessor post-open hooks at 1733246212413 (+6 ms)Region opened successfully at 1733246212413 2024-12-03T17:16:52,415 INFO [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., pid=55, masterSystemTime=1733246212391 2024-12-03T17:16:52,417 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:16:52,417 INFO [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:16:52,417 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=f5a2a3f1964bd8ecfa7908952eaec708, regionState=OPEN, openSeqNum=2, regionLocation=27e0f852a164,41167,1733246151593 2024-12-03T17:16:52,419 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=54, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 because future has completed 2024-12-03T17:16:52,421 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-12-03T17:16:52,421 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; OpenRegionProcedure f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 in 181 msec 2024-12-03T17:16:52,423 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=53 2024-12-03T17:16:52,423 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=53, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f5a2a3f1964bd8ecfa7908952eaec708, ASSIGN in 338 msec 2024-12-03T17:16:52,424 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T17:16:52,424 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733246212424"}]},"ts":"1733246212424"} 2024-12-03T17:16:52,427 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-03T17:16:52,428 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T17:16:52,430 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 897 msec 2024-12-03T17:16:52,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-12-03T17:16:52,674 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:16:52,675 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50502613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:52,675 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:16:52,676 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:16:52,677 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:16:52,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:16:52,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:16:52,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56a35b0e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:52,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:16:52,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:16:52,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:52,678 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47846, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:16:52,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79bab3a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:52,679 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:16:52,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:16:52,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:16:52,682 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42056, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:16:52,683 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T17:16:52,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 27e0f852a164,38523,1733246150595 2024-12-03T17:16:52,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@79a86dc5 2024-12-03T17:16:52,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T17:16:52,685 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47856, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T17:16:52,687 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72325ab8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:52,687 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:16:52,687 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:16:52,690 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:16:52,692 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:16:52,692 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:16:52,692 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66e24754, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:52,693 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:16:52,693 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:16:52,693 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:52,694 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47872, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:16:52,695 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@234ffb2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:52,696 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6686bc48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:52,696 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:16:52,696 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:16:52,697 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:16:52,697 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:16:52,697 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:16:52,698 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28a76607, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:52,698 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:16:52,698 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:16:52,698 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:52,699 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47898, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:16:52,699 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@295a8238, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:52,700 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d70004e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:52,700 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:16:52,700 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:16:52,701 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:16:52,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:16:52,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:16:52,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e7f4eff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:52,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:16:52,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:16:52,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:52,703 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47916, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:16:52,703 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cd4eb25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:52,706 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19903550, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:52,706 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:16:52,707 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:16:52,708 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:16:52,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:16:52,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:16:52,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3afb502, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:52,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:16:52,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:16:52,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:52,710 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47934, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:16:52,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e18fb9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:52,715 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e8815e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:52,715 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:16:52,715 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:16:52,716 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:16:52,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:16:52,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:16:52,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e65af61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:52,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:16:52,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:16:52,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:52,718 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47956, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:16:52,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ade8efc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:52,721 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2611025, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:52,721 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:16:52,722 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:16:52,723 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:16:52,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:16:52,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:16:52,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50442f1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:52,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:16:52,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:16:52,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:52,725 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:16:52,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@396ac6ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:52,731 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8a8ae50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:52,731 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:16:52,731 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:16:52,733 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:16:52,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:16:52,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:16:52,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@321fe104, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:52,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:16:52,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:16:52,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:52,736 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47988, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:16:52,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e3184b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:52,737 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@471771d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:52,737 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:16:52,738 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:16:52,738 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:16:52,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:16:52,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:16:52,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28e6dc80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:52,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:16:52,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:16:52,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:52,740 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48006, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:16:52,741 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@194ce69c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:52,742 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40916ac2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:52,742 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:16:52,742 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:16:52,743 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:16:52,743 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:16:52,743 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:16:52,743 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cb0b20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:52,743 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:16:52,743 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:16:52,743 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:52,744 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48032, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:16:52,745 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@599bd97d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:52,746 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12c47ac0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:52,746 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:16:52,746 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:16:52,747 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:16:52,747 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:16:52,747 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:16:52,747 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ba1a484, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:52,747 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:16:52,748 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:16:52,748 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:16:52,748 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48040, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:16:52,749 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79df999a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:16:52,750 DEBUG [Thread-942 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:16:52,751 DEBUG [Thread-934 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:16:52,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:16:52,751 DEBUG [Thread-949 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:16:52,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:16:52,752 DEBUG [Thread-938 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:16:52,752 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T17:16:52,753 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42066, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:16:52,753 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:16:52,754 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:16:52,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-12-03T17:16:52,754 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:16:52,755 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:16:52,755 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42074, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:16:52,756 DEBUG [Thread-940 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:16:52,756 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42088, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:16:52,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-03T17:16:52,758 DEBUG [Thread-951 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:16:52,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2] 2024-12-03T17:16:52,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2] 2024-12-03T17:16:52,758 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T17:16:52,758 DEBUG [Thread-945 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:16:52,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:16:52,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:16:52,759 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T17:16:52,760 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T17:16:52,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:16:52,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:16:52,761 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2] 2024-12-03T17:16:52,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:16:52,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:16:52,762 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42102, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:16:52,763 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42120, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:16:52,763 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42110, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:16:52,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2] 2024-12-03T17:16:52,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2] 2024-12-03T17:16:52,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2] 2024-12-03T17:16:52,768 DEBUG [Thread-951 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:16:52,770 DEBUG [Thread-951 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:16:52,770 DEBUG [Thread-947 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:16:52,770 DEBUG [Thread-936 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:16:52,771 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:16:52,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on f5a2a3f1964bd8ecfa7908952eaec708 2024-12-03T17:16:52,771 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f5a2a3f1964bd8ecfa7908952eaec708 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T17:16:52,771 DEBUG [Thread-936 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:16:52,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:16:52,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:16:52,773 DEBUG [Thread-953 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:16:52,774 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42134, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:16:52,774 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42164, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:16:52,774 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42132, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:16:52,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=A 2024-12-03T17:16:52,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:52,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=B 2024-12-03T17:16:52,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:52,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=C 2024-12-03T17:16:52,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:52,775 DEBUG [Thread-953 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:16:52,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2] 2024-12-03T17:16:52,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2] 2024-12-03T17:16:52,781 DEBUG [Thread-953 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:16:52,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2] 2024-12-03T17:16:52,787 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42176, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:16:52,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2] 2024-12-03T17:16:52,809 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/f2f2f8b7324943c08d4248641513361b is 50, key is test_row_0/A:col10/1733246212769/Put/seqid=0 2024-12-03T17:16:52,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742019_1195 (size=12001) 2024-12-03T17:16:52,824 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/f2f2f8b7324943c08d4248641513361b 2024-12-03T17:16:52,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733246212806, max timestamp=1733246212823Num uniques 859; 2024-12-03T17:16:52,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T17:16:52,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733246212806, max timestamp=1733246212823Num uniques 859; 2024-12-03T17:16:52,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T17:16:52,831 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733246212806, max timestamp=1733246212827Num uniques 859; 2024-12-03T17:16:52,831 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T17:16:52,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-03T17:16:52,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:52,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:52,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246272873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:52,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42132 deadline: 1733246272873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:52,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:52,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:52,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:52,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:52,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:52,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:52,887 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/44016882368743f18d498a980d752fc9 is 50, key is test_row_0/B:col10/1733246212769/Put/seqid=0 2024-12-03T17:16:52,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:52,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246272876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:52,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:52,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:52,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:52,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:52,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246272889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:52,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:52,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:52,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:52,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246272891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:52,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:52,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:52,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:52,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:52,914 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-03T17:16:52,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:16:52,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. as already flushing 2024-12-03T17:16:52,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:16:52,916 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:52,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:52,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:52,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742020_1196 (size=12001) 2024-12-03T17:16:52,946 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/44016882368743f18d498a980d752fc9 2024-12-03T17:16:53,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:53,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246272993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:53,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:53,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42132 deadline: 1733246272994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:53,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:53,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:53,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:53,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:53,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246273005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:53,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:53,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:53,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:53,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246273005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:53,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:53,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:53,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246273004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:53,009 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:53,009 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:53,009 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:53,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:53,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:53,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:53,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:53,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:53,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:53,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/735f576162194064986a40028fa8098a is 50, key is test_row_0/C:col10/1733246212769/Put/seqid=0 2024-12-03T17:16:53,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742021_1197 (size=12001) 2024-12-03T17:16:53,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-03T17:16:53,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-03T17:16:53,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:16:53,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. as already flushing 2024-12-03T17:16:53,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:16:53,075 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:53,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:53,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:53,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:53,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246273214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:53,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:53,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:53,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:53,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:53,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246273215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:53,222 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:53,222 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:53,222 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:53,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:53,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246273215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:53,223 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:53,223 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:53,223 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:53,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:53,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246273215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:53,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:53,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:53,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:53,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:53,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42132 deadline: 1733246273219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:53,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:53,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:53,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:53,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-03T17:16:53,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:16:53,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. as already flushing 2024-12-03T17:16:53,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:16:53,232 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:53,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:53,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:53,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-03T17:16:53,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-03T17:16:53,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:16:53,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. as already flushing 2024-12-03T17:16:53,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:16:53,394 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:53,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:53,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:53,482 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/735f576162194064986a40028fa8098a 2024-12-03T17:16:53,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:53,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246273534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:53,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:53,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246273534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:53,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:53,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:53,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:53,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:53,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:53,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246273535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:53,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246273534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:53,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:53,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:53,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42132 deadline: 1733246273537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:53,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:53,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:53,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:53,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:53,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:53,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:53,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:53,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:53,542 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:53,542 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:53,542 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:53,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/f2f2f8b7324943c08d4248641513361b as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/f2f2f8b7324943c08d4248641513361b 2024-12-03T17:16:53,554 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/f2f2f8b7324943c08d4248641513361b, entries=150, sequenceid=11, filesize=11.7 K 2024-12-03T17:16:53,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/44016882368743f18d498a980d752fc9 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/44016882368743f18d498a980d752fc9 2024-12-03T17:16:53,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-03T17:16:53,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:16:53,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. as already flushing 2024-12-03T17:16:53,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:16:53,562 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:53,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:53,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:53,568 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/44016882368743f18d498a980d752fc9, entries=150, sequenceid=11, filesize=11.7 K 2024-12-03T17:16:53,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/735f576162194064986a40028fa8098a as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/735f576162194064986a40028fa8098a 2024-12-03T17:16:53,604 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/735f576162194064986a40028fa8098a, entries=150, sequenceid=11, filesize=11.7 K 2024-12-03T17:16:53,606 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=143.13 KB/146560 for f5a2a3f1964bd8ecfa7908952eaec708 in 835ms, sequenceid=11, compaction requested=false 2024-12-03T17:16:53,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:16:53,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-03T17:16:53,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:16:53,717 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing f5a2a3f1964bd8ecfa7908952eaec708 3/3 column families, dataSize=143.13 KB heapSize=405.30 KB 2024-12-03T17:16:53,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=A 2024-12-03T17:16:53,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:53,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=B 2024-12-03T17:16:53,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:53,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=C 2024-12-03T17:16:53,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:53,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/c6b1f679a8ae4ba39a4e9eacf058f1b8 is 50, key is test_row_0/A:col10/1733246212873/Put/seqid=0 2024-12-03T17:16:53,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742022_1198 (size=12001) 2024-12-03T17:16:53,766 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/c6b1f679a8ae4ba39a4e9eacf058f1b8 2024-12-03T17:16:53,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/f352954592ce42de8bd97ad80267a801 is 50, key is test_row_0/B:col10/1733246212873/Put/seqid=0 2024-12-03T17:16:53,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742023_1199 (size=12001) 2024-12-03T17:16:53,824 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/f352954592ce42de8bd97ad80267a801 2024-12-03T17:16:53,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/adabd2f6ca7e4a8d95bde30eaa48e59b is 50, key is test_row_0/C:col10/1733246212873/Put/seqid=0 2024-12-03T17:16:53,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-03T17:16:53,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742024_1200 (size=12001) 2024-12-03T17:16:53,901 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/adabd2f6ca7e4a8d95bde30eaa48e59b 2024-12-03T17:16:53,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/c6b1f679a8ae4ba39a4e9eacf058f1b8 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/c6b1f679a8ae4ba39a4e9eacf058f1b8 2024-12-03T17:16:53,943 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/c6b1f679a8ae4ba39a4e9eacf058f1b8, entries=150, sequenceid=48, filesize=11.7 K 2024-12-03T17:16:53,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/f352954592ce42de8bd97ad80267a801 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/f352954592ce42de8bd97ad80267a801 2024-12-03T17:16:53,962 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/f352954592ce42de8bd97ad80267a801, entries=150, sequenceid=48, filesize=11.7 K 2024-12-03T17:16:53,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/adabd2f6ca7e4a8d95bde30eaa48e59b as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/adabd2f6ca7e4a8d95bde30eaa48e59b 2024-12-03T17:16:53,987 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/adabd2f6ca7e4a8d95bde30eaa48e59b, entries=150, sequenceid=48, filesize=11.7 K 2024-12-03T17:16:53,989 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~143.13 KB/146560, heapSize ~405.25 KB/414976, currentSize=0 B/0 for f5a2a3f1964bd8ecfa7908952eaec708 in 273ms, sequenceid=48, compaction requested=false 2024-12-03T17:16:53,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:16:53,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:16:53,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-12-03T17:16:53,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-12-03T17:16:53,997 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-12-03T17:16:53,997 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2360 sec 2024-12-03T17:16:53,999 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 1.2460 sec 2024-12-03T17:16:54,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on f5a2a3f1964bd8ecfa7908952eaec708 2024-12-03T17:16:54,065 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f5a2a3f1964bd8ecfa7908952eaec708 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T17:16:54,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=A 2024-12-03T17:16:54,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:54,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=B 2024-12-03T17:16:54,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:54,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=C 2024-12-03T17:16:54,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:54,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/0de1fb8f9ad747e8a6a6aaf432a4c37e is 50, key is test_row_0/A:col10/1733246214062/Put/seqid=0 2024-12-03T17:16:54,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742025_1201 (size=16681) 2024-12-03T17:16:54,104 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/0de1fb8f9ad747e8a6a6aaf432a4c37e 2024-12-03T17:16:54,120 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/e997d37380d04c96b9c978121c5dd6ee is 50, key is test_row_0/B:col10/1733246214062/Put/seqid=0 2024-12-03T17:16:54,131 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733246214097, max timestamp=1733246214118Num uniques 859; 2024-12-03T17:16:54,131 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T17:16:54,132 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733246214097, max timestamp=1733246214118Num uniques 859; 2024-12-03T17:16:54,132 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T17:16:54,133 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733246214097, max timestamp=1733246214126Num uniques 859; 2024-12-03T17:16:54,133 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T17:16:54,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742026_1202 (size=12001) 2024-12-03T17:16:54,169 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/e997d37380d04c96b9c978121c5dd6ee 2024-12-03T17:16:54,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:54,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246274174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:54,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:54,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:54,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:54,189 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/9a41d48170ba4c219358c6f5a22e3063 is 50, key is test_row_0/C:col10/1733246214062/Put/seqid=0 2024-12-03T17:16:54,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:54,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246274178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:54,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:54,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:54,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:54,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:54,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246274182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:54,190 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:54,190 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:54,190 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:54,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:54,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246274182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:54,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:54,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:54,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:54,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:54,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42132 deadline: 1733246274186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:54,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:54,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:54,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:54,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742027_1203 (size=12001) 2024-12-03T17:16:54,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:54,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246274294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:54,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:54,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:54,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:54,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246274294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:54,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:54,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:54,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:54,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:54,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:54,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246274294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:54,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:54,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:54,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:54,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:54,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246274304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:54,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:54,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:54,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:54,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:54,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42132 deadline: 1733246274304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:54,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:54,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:54,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:54,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:54,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246274504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:54,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:54,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:54,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:54,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:54,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246274505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:54,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:54,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:54,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:54,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246274505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:54,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:54,511 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:54,511 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:54,511 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:54,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:54,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246274513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:54,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:54,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:54,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:54,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:54,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42132 deadline: 1733246274524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:54,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:54,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:54,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:54,639 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/9a41d48170ba4c219358c6f5a22e3063 2024-12-03T17:16:54,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/0de1fb8f9ad747e8a6a6aaf432a4c37e as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/0de1fb8f9ad747e8a6a6aaf432a4c37e 2024-12-03T17:16:54,657 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/0de1fb8f9ad747e8a6a6aaf432a4c37e, entries=250, sequenceid=60, filesize=16.3 K 2024-12-03T17:16:54,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/e997d37380d04c96b9c978121c5dd6ee as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/e997d37380d04c96b9c978121c5dd6ee 2024-12-03T17:16:54,668 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/e997d37380d04c96b9c978121c5dd6ee, entries=150, sequenceid=60, filesize=11.7 K 2024-12-03T17:16:54,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/9a41d48170ba4c219358c6f5a22e3063 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/9a41d48170ba4c219358c6f5a22e3063 2024-12-03T17:16:54,690 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/9a41d48170ba4c219358c6f5a22e3063, entries=150, sequenceid=60, filesize=11.7 K 2024-12-03T17:16:54,691 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for f5a2a3f1964bd8ecfa7908952eaec708 in 625ms, sequenceid=60, compaction requested=true 2024-12-03T17:16:54,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:16:54,691 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:16:54,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5a2a3f1964bd8ecfa7908952eaec708:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T17:16:54,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:54,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5a2a3f1964bd8ecfa7908952eaec708:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T17:16:54,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:54,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5a2a3f1964bd8ecfa7908952eaec708:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T17:16:54,691 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:16:54,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:16:54,693 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:16:54,693 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): f5a2a3f1964bd8ecfa7908952eaec708/A is initiating minor compaction (all files) 2024-12-03T17:16:54,693 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:16:54,693 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5a2a3f1964bd8ecfa7908952eaec708/A in TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:16:54,693 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): f5a2a3f1964bd8ecfa7908952eaec708/B is initiating minor compaction (all files) 2024-12-03T17:16:54,693 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5a2a3f1964bd8ecfa7908952eaec708/B in TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:16:54,693 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/f2f2f8b7324943c08d4248641513361b, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/c6b1f679a8ae4ba39a4e9eacf058f1b8, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/0de1fb8f9ad747e8a6a6aaf432a4c37e] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp, totalSize=39.7 K 2024-12-03T17:16:54,693 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/44016882368743f18d498a980d752fc9, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/f352954592ce42de8bd97ad80267a801, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/e997d37380d04c96b9c978121c5dd6ee] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp, totalSize=35.2 K 2024-12-03T17:16:54,693 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2f2f8b7324943c08d4248641513361b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733246212766 2024-12-03T17:16:54,693 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 44016882368743f18d498a980d752fc9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733246212766 2024-12-03T17:16:54,694 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting f352954592ce42de8bd97ad80267a801, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733246212873 2024-12-03T17:16:54,694 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6b1f679a8ae4ba39a4e9eacf058f1b8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733246212873 2024-12-03T17:16:54,694 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting e997d37380d04c96b9c978121c5dd6ee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733246214062 2024-12-03T17:16:54,694 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0de1fb8f9ad747e8a6a6aaf432a4c37e, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733246214058 2024-12-03T17:16:54,706 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5a2a3f1964bd8ecfa7908952eaec708#B#compaction#162 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:16:54,707 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/99e3f78412914425bac8cf76ebf8ee48 is 50, key is test_row_0/B:col10/1733246214062/Put/seqid=0 2024-12-03T17:16:54,711 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5a2a3f1964bd8ecfa7908952eaec708#A#compaction#163 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:16:54,712 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/582fb99ef9f84933a179be6da7b5192d is 50, key is test_row_0/A:col10/1733246214062/Put/seqid=0 2024-12-03T17:16:54,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742028_1204 (size=12104) 2024-12-03T17:16:54,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742029_1205 (size=12104) 2024-12-03T17:16:54,782 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/99e3f78412914425bac8cf76ebf8ee48 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/99e3f78412914425bac8cf76ebf8ee48 2024-12-03T17:16:54,786 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/582fb99ef9f84933a179be6da7b5192d as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/582fb99ef9f84933a179be6da7b5192d 2024-12-03T17:16:54,799 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5a2a3f1964bd8ecfa7908952eaec708/B of f5a2a3f1964bd8ecfa7908952eaec708 into 99e3f78412914425bac8cf76ebf8ee48(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:16:54,799 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:16:54,799 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., storeName=f5a2a3f1964bd8ecfa7908952eaec708/B, priority=13, startTime=1733246214691; duration=0sec 2024-12-03T17:16:54,800 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:16:54,800 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5a2a3f1964bd8ecfa7908952eaec708:B 2024-12-03T17:16:54,800 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:16:54,803 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:16:54,803 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): f5a2a3f1964bd8ecfa7908952eaec708/C is initiating minor compaction (all files) 2024-12-03T17:16:54,803 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5a2a3f1964bd8ecfa7908952eaec708/C in TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:16:54,803 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/735f576162194064986a40028fa8098a, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/adabd2f6ca7e4a8d95bde30eaa48e59b, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/9a41d48170ba4c219358c6f5a22e3063] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp, totalSize=35.2 K 2024-12-03T17:16:54,804 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 735f576162194064986a40028fa8098a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733246212766 2024-12-03T17:16:54,804 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting adabd2f6ca7e4a8d95bde30eaa48e59b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733246212873 2024-12-03T17:16:54,805 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a41d48170ba4c219358c6f5a22e3063, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733246214062 2024-12-03T17:16:54,808 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5a2a3f1964bd8ecfa7908952eaec708/A of f5a2a3f1964bd8ecfa7908952eaec708 into 582fb99ef9f84933a179be6da7b5192d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:16:54,808 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:16:54,808 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., storeName=f5a2a3f1964bd8ecfa7908952eaec708/A, priority=13, startTime=1733246214691; duration=0sec 2024-12-03T17:16:54,809 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:54,809 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5a2a3f1964bd8ecfa7908952eaec708:A 2024-12-03T17:16:54,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on f5a2a3f1964bd8ecfa7908952eaec708 2024-12-03T17:16:54,820 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f5a2a3f1964bd8ecfa7908952eaec708 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-03T17:16:54,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=A 2024-12-03T17:16:54,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:54,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=B 2024-12-03T17:16:54,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:54,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=C 2024-12-03T17:16:54,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:54,821 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5a2a3f1964bd8ecfa7908952eaec708#C#compaction#164 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:16:54,821 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/55efcb7a565d49f38f2e216350c472ac is 50, key is test_row_0/C:col10/1733246214062/Put/seqid=0 2024-12-03T17:16:54,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/cfb11057901e43659c5de495a3c289de is 50, key is test_row_0/A:col10/1733246214818/Put/seqid=0 2024-12-03T17:16:54,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742030_1206 (size=12104) 2024-12-03T17:16:54,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:54,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246274848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:54,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:54,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:54,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:54,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:54,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42132 deadline: 1733246274849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:54,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:54,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:54,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:54,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:54,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246274850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:54,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:54,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:54,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:54,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:54,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246274850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:54,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:54,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:54,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:54,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:54,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246274851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:54,867 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:54,868 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:54,868 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:54,869 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/55efcb7a565d49f38f2e216350c472ac as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/55efcb7a565d49f38f2e216350c472ac 2024-12-03T17:16:54,877 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5a2a3f1964bd8ecfa7908952eaec708/C of f5a2a3f1964bd8ecfa7908952eaec708 into 55efcb7a565d49f38f2e216350c472ac(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:16:54,877 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:16:54,877 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., storeName=f5a2a3f1964bd8ecfa7908952eaec708/C, priority=13, startTime=1733246214691; duration=0sec 2024-12-03T17:16:54,877 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:54,877 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5a2a3f1964bd8ecfa7908952eaec708:C 2024-12-03T17:16:54,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742031_1207 (size=14341) 2024-12-03T17:16:54,881 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/cfb11057901e43659c5de495a3c289de 2024-12-03T17:16:54,910 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/5b0cea4aedeb483995ce7116e6694d62 is 50, key is test_row_0/B:col10/1733246214818/Put/seqid=0 2024-12-03T17:16:54,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-03T17:16:54,915 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:16:54,919 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T17:16:54,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-12-03T17:16:54,921 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T17:16:54,923 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T17:16:54,924 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T17:16:54,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-03T17:16:54,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742032_1208 (size=12001) 2024-12-03T17:16:54,931 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/5b0cea4aedeb483995ce7116e6694d62 2024-12-03T17:16:54,945 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/6e66da2558bc401f8cabac5582cf7083 is 50, key is test_row_0/C:col10/1733246214818/Put/seqid=0 2024-12-03T17:16:54,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:54,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246274966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:54,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742033_1209 (size=12001) 2024-12-03T17:16:54,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:54,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42132 deadline: 1733246274966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:54,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:54,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:54,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:54,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:54,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:54,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:54,971 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/6e66da2558bc401f8cabac5582cf7083 2024-12-03T17:16:54,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/cfb11057901e43659c5de495a3c289de as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/cfb11057901e43659c5de495a3c289de 2024-12-03T17:16:54,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:54,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246274974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:54,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:54,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:54,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:54,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:54,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246274974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:54,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:54,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:54,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:54,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:54,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246274974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:54,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:54,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:54,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:54,987 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/cfb11057901e43659c5de495a3c289de, entries=200, sequenceid=98, filesize=14.0 K 2024-12-03T17:16:54,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/5b0cea4aedeb483995ce7116e6694d62 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/5b0cea4aedeb483995ce7116e6694d62 2024-12-03T17:16:55,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/5b0cea4aedeb483995ce7116e6694d62, entries=150, sequenceid=98, filesize=11.7 K 2024-12-03T17:16:55,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/6e66da2558bc401f8cabac5582cf7083 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/6e66da2558bc401f8cabac5582cf7083 2024-12-03T17:16:55,034 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/6e66da2558bc401f8cabac5582cf7083, entries=150, sequenceid=98, filesize=11.7 K 2024-12-03T17:16:55,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-03T17:16:55,035 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=67.09 KB/68700 for f5a2a3f1964bd8ecfa7908952eaec708 in 215ms, sequenceid=98, compaction requested=false 2024-12-03T17:16:55,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:16:55,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-03T17:16:55,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:16:55,088 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing f5a2a3f1964bd8ecfa7908952eaec708 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T17:16:55,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=A 2024-12-03T17:16:55,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:55,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=B 2024-12-03T17:16:55,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:55,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=C 2024-12-03T17:16:55,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:55,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/5b62d3fbb9a24130865294f06fe6bd43 is 50, key is test_row_0/A:col10/1733246214844/Put/seqid=0 2024-12-03T17:16:55,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742034_1210 (size=12001) 2024-12-03T17:16:55,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. as already flushing 2024-12-03T17:16:55,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on f5a2a3f1964bd8ecfa7908952eaec708 2024-12-03T17:16:55,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-03T17:16:55,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:55,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42132 deadline: 1733246275242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:55,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:55,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:55,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:55,256 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T17:16:55,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:55,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246275250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:55,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:55,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246275251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:55,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:55,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:55,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246275252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:55,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:55,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:55,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:55,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:55,261 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:55,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:55,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246275255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:55,261 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:55,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:55,261 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:55,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:55,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:55,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:55,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:55,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246275363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:55,372 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:55,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:55,372 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:55,372 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:55,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42132 deadline: 1733246275364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:55,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:55,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:55,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:55,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:55,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246275364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:55,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:55,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246275364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:55,373 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:55,373 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:55,373 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:55,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:55,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:55,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:55,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:55,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246275371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:55,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:55,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:55,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:55,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-03T17:16:55,567 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/5b62d3fbb9a24130865294f06fe6bd43 2024-12-03T17:16:55,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:55,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246275586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:55,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:55,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:55,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:55,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:55,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:55,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42132 deadline: 1733246275586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:55,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:55,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:55,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:55,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:55,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246275586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:55,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:55,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246275587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:55,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:55,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:55,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:55,597 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:55,597 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:55,597 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:55,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246275586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:55,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:55,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:55,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:55,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/3ba29def7138414d873495722cb9adf5 is 50, key is test_row_0/B:col10/1733246214844/Put/seqid=0 2024-12-03T17:16:55,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742035_1211 (size=12001) 2024-12-03T17:16:55,667 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/3ba29def7138414d873495722cb9adf5 2024-12-03T17:16:55,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/ea747c218a8843348d4dbcb157b352c0 is 50, key is test_row_0/C:col10/1733246214844/Put/seqid=0 2024-12-03T17:16:55,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742036_1212 (size=12001) 2024-12-03T17:16:55,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:55,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246275904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:55,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:55,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:55,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:55,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246275904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:55,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:55,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:55,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:55,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246275904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:55,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:55,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:55,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:55,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:55,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42132 deadline: 1733246275904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:55,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:55,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:55,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:55,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:55,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:55,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:55,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246275913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:55,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:55,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:55,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:56,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-03T17:16:56,136 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/ea747c218a8843348d4dbcb157b352c0 2024-12-03T17:16:56,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/5b62d3fbb9a24130865294f06fe6bd43 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/5b62d3fbb9a24130865294f06fe6bd43 2024-12-03T17:16:56,165 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/5b62d3fbb9a24130865294f06fe6bd43, entries=150, sequenceid=112, filesize=11.7 K 2024-12-03T17:16:56,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/3ba29def7138414d873495722cb9adf5 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/3ba29def7138414d873495722cb9adf5 2024-12-03T17:16:56,176 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/3ba29def7138414d873495722cb9adf5, entries=150, sequenceid=112, filesize=11.7 K 2024-12-03T17:16:56,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/ea747c218a8843348d4dbcb157b352c0 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/ea747c218a8843348d4dbcb157b352c0 2024-12-03T17:16:56,201 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/ea747c218a8843348d4dbcb157b352c0, entries=150, sequenceid=112, filesize=11.7 K 2024-12-03T17:16:56,226 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for f5a2a3f1964bd8ecfa7908952eaec708 in 1138ms, sequenceid=112, compaction requested=true 2024-12-03T17:16:56,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:16:56,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:16:56,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-12-03T17:16:56,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-12-03T17:16:56,245 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-12-03T17:16:56,245 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3100 sec 2024-12-03T17:16:56,247 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 1.3270 sec 2024-12-03T17:16:56,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on f5a2a3f1964bd8ecfa7908952eaec708 2024-12-03T17:16:56,421 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f5a2a3f1964bd8ecfa7908952eaec708 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T17:16:56,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=A 2024-12-03T17:16:56,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:56,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=B 2024-12-03T17:16:56,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:56,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=C 2024-12-03T17:16:56,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:56,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/465ed8ccd6c5415b9f1897267c2d7cff is 50, key is test_row_0/A:col10/1733246215215/Put/seqid=0 2024-12-03T17:16:56,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742037_1213 (size=12101) 2024-12-03T17:16:56,460 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/465ed8ccd6c5415b9f1897267c2d7cff 2024-12-03T17:16:56,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:56,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246276453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:56,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:56,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246276460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:56,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:56,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:56,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:56,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:56,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42132 deadline: 1733246276461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:56,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:56,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:56,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:56,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:56,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:56,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:56,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:56,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246276486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:56,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:56,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:56,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:56,501 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/27c369e20f62429f8b14f1e7259a7fd7 is 50, key is test_row_0/B:col10/1733246215215/Put/seqid=0 2024-12-03T17:16:56,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:56,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246276492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:56,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:56,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:56,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:56,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742038_1214 (size=12101) 2024-12-03T17:16:56,538 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/27c369e20f62429f8b14f1e7259a7fd7 2024-12-03T17:16:56,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/3baf1b2321314a13adf325431c00fd6e is 50, key is test_row_0/C:col10/1733246215215/Put/seqid=0 2024-12-03T17:16:56,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:56,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246276587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:56,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:56,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246276591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:56,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:56,597 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:56,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:56,597 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:56,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:56,597 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:56,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:56,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42132 deadline: 1733246276593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:56,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:56,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:56,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742039_1215 (size=12101) 2024-12-03T17:16:56,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:56,602 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/3baf1b2321314a13adf325431c00fd6e 2024-12-03T17:16:56,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:56,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246276604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:56,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:56,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:56,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:56,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/465ed8ccd6c5415b9f1897267c2d7cff as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/465ed8ccd6c5415b9f1897267c2d7cff 2024-12-03T17:16:56,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:56,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246276615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:56,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:56,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:56,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:56,628 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/465ed8ccd6c5415b9f1897267c2d7cff, entries=150, sequenceid=133, filesize=11.8 K 2024-12-03T17:16:56,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/27c369e20f62429f8b14f1e7259a7fd7 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/27c369e20f62429f8b14f1e7259a7fd7 2024-12-03T17:16:56,643 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/27c369e20f62429f8b14f1e7259a7fd7, entries=150, sequenceid=133, filesize=11.8 K 2024-12-03T17:16:56,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/3baf1b2321314a13adf325431c00fd6e as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/3baf1b2321314a13adf325431c00fd6e 2024-12-03T17:16:56,652 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/3baf1b2321314a13adf325431c00fd6e, entries=150, sequenceid=133, filesize=11.8 K 2024-12-03T17:16:56,658 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=60.38 KB/61830 for f5a2a3f1964bd8ecfa7908952eaec708 in 237ms, sequenceid=133, compaction requested=true 2024-12-03T17:16:56,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:16:56,659 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T17:16:56,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5a2a3f1964bd8ecfa7908952eaec708:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T17:16:56,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:56,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5a2a3f1964bd8ecfa7908952eaec708:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T17:16:56,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:56,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5a2a3f1964bd8ecfa7908952eaec708:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T17:16:56,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:16:56,659 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T17:16:56,661 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50547 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T17:16:56,662 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): f5a2a3f1964bd8ecfa7908952eaec708/A is initiating minor compaction (all files) 2024-12-03T17:16:56,662 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5a2a3f1964bd8ecfa7908952eaec708/A in TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:16:56,662 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/582fb99ef9f84933a179be6da7b5192d, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/cfb11057901e43659c5de495a3c289de, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/5b62d3fbb9a24130865294f06fe6bd43, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/465ed8ccd6c5415b9f1897267c2d7cff] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp, totalSize=49.4 K 2024-12-03T17:16:56,662 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48207 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T17:16:56,662 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): f5a2a3f1964bd8ecfa7908952eaec708/B is initiating minor compaction (all files) 2024-12-03T17:16:56,662 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5a2a3f1964bd8ecfa7908952eaec708/B in TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:16:56,662 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 582fb99ef9f84933a179be6da7b5192d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733246214062 2024-12-03T17:16:56,662 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/99e3f78412914425bac8cf76ebf8ee48, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/5b0cea4aedeb483995ce7116e6694d62, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/3ba29def7138414d873495722cb9adf5, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/27c369e20f62429f8b14f1e7259a7fd7] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp, totalSize=47.1 K 2024-12-03T17:16:56,663 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfb11057901e43659c5de495a3c289de, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733246214171 2024-12-03T17:16:56,664 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b62d3fbb9a24130865294f06fe6bd43, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733246214844 2024-12-03T17:16:56,664 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 99e3f78412914425bac8cf76ebf8ee48, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733246214062 2024-12-03T17:16:56,664 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 465ed8ccd6c5415b9f1897267c2d7cff, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733246215215 2024-12-03T17:16:56,664 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b0cea4aedeb483995ce7116e6694d62, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733246214171 2024-12-03T17:16:56,665 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ba29def7138414d873495722cb9adf5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733246214844 2024-12-03T17:16:56,667 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 27c369e20f62429f8b14f1e7259a7fd7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733246215215 2024-12-03T17:16:56,681 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5a2a3f1964bd8ecfa7908952eaec708#B#compaction#175 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:16:56,681 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/12b7b6fd1ea3404a85e80c7fb8f34bb5 is 50, key is test_row_0/B:col10/1733246215215/Put/seqid=0 2024-12-03T17:16:56,682 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5a2a3f1964bd8ecfa7908952eaec708#A#compaction#174 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:16:56,682 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/ad1c140c58e74febb55311230e64e66a is 50, key is test_row_0/A:col10/1733246215215/Put/seqid=0 2024-12-03T17:16:56,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742041_1217 (size=12341) 2024-12-03T17:16:56,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742040_1216 (size=12341) 2024-12-03T17:16:56,747 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/ad1c140c58e74febb55311230e64e66a as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/ad1c140c58e74febb55311230e64e66a 2024-12-03T17:16:56,751 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/12b7b6fd1ea3404a85e80c7fb8f34bb5 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/12b7b6fd1ea3404a85e80c7fb8f34bb5 2024-12-03T17:16:56,763 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f5a2a3f1964bd8ecfa7908952eaec708/A of f5a2a3f1964bd8ecfa7908952eaec708 into ad1c140c58e74febb55311230e64e66a(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:16:56,763 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:16:56,763 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., storeName=f5a2a3f1964bd8ecfa7908952eaec708/A, priority=12, startTime=1733246216659; duration=0sec 2024-12-03T17:16:56,763 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:16:56,763 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5a2a3f1964bd8ecfa7908952eaec708:A 2024-12-03T17:16:56,763 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T17:16:56,764 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f5a2a3f1964bd8ecfa7908952eaec708/B of f5a2a3f1964bd8ecfa7908952eaec708 into 12b7b6fd1ea3404a85e80c7fb8f34bb5(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:16:56,764 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:16:56,764 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., storeName=f5a2a3f1964bd8ecfa7908952eaec708/B, priority=12, startTime=1733246216659; duration=0sec 2024-12-03T17:16:56,764 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:56,764 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5a2a3f1964bd8ecfa7908952eaec708:B 2024-12-03T17:16:56,771 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48207 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T17:16:56,771 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): f5a2a3f1964bd8ecfa7908952eaec708/C is initiating minor compaction (all files) 2024-12-03T17:16:56,771 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5a2a3f1964bd8ecfa7908952eaec708/C in TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:16:56,771 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/55efcb7a565d49f38f2e216350c472ac, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/6e66da2558bc401f8cabac5582cf7083, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/ea747c218a8843348d4dbcb157b352c0, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/3baf1b2321314a13adf325431c00fd6e] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp, totalSize=47.1 K 2024-12-03T17:16:56,774 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55efcb7a565d49f38f2e216350c472ac, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733246214062 2024-12-03T17:16:56,774 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e66da2558bc401f8cabac5582cf7083, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733246214171 2024-12-03T17:16:56,776 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea747c218a8843348d4dbcb157b352c0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733246214844 2024-12-03T17:16:56,776 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3baf1b2321314a13adf325431c00fd6e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733246215215 2024-12-03T17:16:56,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on f5a2a3f1964bd8ecfa7908952eaec708 2024-12-03T17:16:56,808 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f5a2a3f1964bd8ecfa7908952eaec708 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T17:16:56,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=A 2024-12-03T17:16:56,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:56,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=B 2024-12-03T17:16:56,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:56,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=C 2024-12-03T17:16:56,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:56,809 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5a2a3f1964bd8ecfa7908952eaec708#C#compaction#176 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:16:56,810 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/6977b13fc6b34ff8960037134802ac29 is 50, key is test_row_0/C:col10/1733246215215/Put/seqid=0 2024-12-03T17:16:56,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/0b77a685fb7a423d8903655a55e3a13c is 50, key is test_row_0/A:col10/1733246216470/Put/seqid=0 2024-12-03T17:16:56,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742042_1218 (size=12341) 2024-12-03T17:16:56,840 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/6977b13fc6b34ff8960037134802ac29 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/6977b13fc6b34ff8960037134802ac29 2024-12-03T17:16:56,849 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f5a2a3f1964bd8ecfa7908952eaec708/C of f5a2a3f1964bd8ecfa7908952eaec708 into 6977b13fc6b34ff8960037134802ac29(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:16:56,849 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:16:56,849 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., storeName=f5a2a3f1964bd8ecfa7908952eaec708/C, priority=12, startTime=1733246216659; duration=0sec 2024-12-03T17:16:56,849 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:56,849 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5a2a3f1964bd8ecfa7908952eaec708:C 2024-12-03T17:16:56,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742043_1219 (size=12151) 2024-12-03T17:16:56,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:56,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42132 deadline: 1733246276858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:56,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:56,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:56,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:56,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:56,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246276869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:56,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:56,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246276870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:56,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:56,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:56,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246276870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:56,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:56,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:56,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:56,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:56,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:56,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:56,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246276871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:56,880 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:56,880 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:56,880 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:56,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:56,881 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:56,881 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:56,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:56,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42132 deadline: 1733246276974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:56,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:56,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:56,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:56,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:56,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246276984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:56,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:56,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:56,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:56,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:56,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246276984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:56,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:56,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246276985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:56,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:56,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:56,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:56,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:56,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:56,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:56,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:56,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246276985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:56,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:56,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:56,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:57,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-03T17:16:57,084 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:16:57,085 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T17:16:57,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-12-03T17:16:57,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-03T17:16:57,100 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T17:16:57,103 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T17:16:57,104 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T17:16:57,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:57,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42132 deadline: 1733246277185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:57,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:57,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:57,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:57,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-03T17:16:57,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:57,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246277195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:57,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:57,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246277195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:57,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:57,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:57,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:57,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:57,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246277197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:57,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:57,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246277197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:57,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:57,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:57,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:57,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:57,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:57,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:57,202 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:57,202 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:57,202 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:57,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-03T17:16:57,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:16:57,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. as already flushing 2024-12-03T17:16:57,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:16:57,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:57,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:57,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:57,271 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/0b77a685fb7a423d8903655a55e3a13c 2024-12-03T17:16:57,312 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/f079c3008ae84b72875e236b04e02148 is 50, key is test_row_0/B:col10/1733246216470/Put/seqid=0 2024-12-03T17:16:57,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742044_1220 (size=12151) 2024-12-03T17:16:57,376 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/f079c3008ae84b72875e236b04e02148 2024-12-03T17:16:57,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-03T17:16:57,419 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-03T17:16:57,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:16:57,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. as already flushing 2024-12-03T17:16:57,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:16:57,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:57,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:57,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:57,444 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/0bba7d0abee84966b20619340a2a3c93 is 50, key is test_row_0/C:col10/1733246216470/Put/seqid=0 2024-12-03T17:16:57,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742045_1221 (size=12151) 2024-12-03T17:16:57,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:57,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42132 deadline: 1733246277498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:57,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:57,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:57,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:57,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:57,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246277504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:57,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:57,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:57,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:57,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:57,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246277513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:57,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:57,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:57,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:57,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:57,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246277513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:57,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:57,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:57,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:57,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:57,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246277514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:57,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:57,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:57,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:57,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-03T17:16:57,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:16:57,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. as already flushing 2024-12-03T17:16:57,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:16:57,583 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:57,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:57,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:57,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-03T17:16:57,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-03T17:16:57,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:16:57,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. as already flushing 2024-12-03T17:16:57,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:16:57,747 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:57,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:57,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:57,907 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/0bba7d0abee84966b20619340a2a3c93 2024-12-03T17:16:57,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-03T17:16:57,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:16:57,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. as already flushing 2024-12-03T17:16:57,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:16:57,910 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:57,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:57,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:57,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/0b77a685fb7a423d8903655a55e3a13c as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/0b77a685fb7a423d8903655a55e3a13c 2024-12-03T17:16:57,944 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/0b77a685fb7a423d8903655a55e3a13c, entries=150, sequenceid=148, filesize=11.9 K 2024-12-03T17:16:57,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/f079c3008ae84b72875e236b04e02148 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/f079c3008ae84b72875e236b04e02148 2024-12-03T17:16:57,966 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/f079c3008ae84b72875e236b04e02148, entries=150, sequenceid=148, filesize=11.9 K 2024-12-03T17:16:57,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/0bba7d0abee84966b20619340a2a3c93 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/0bba7d0abee84966b20619340a2a3c93 2024-12-03T17:16:57,998 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/0bba7d0abee84966b20619340a2a3c93, entries=150, sequenceid=148, filesize=11.9 K 2024-12-03T17:16:58,003 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for f5a2a3f1964bd8ecfa7908952eaec708 in 1196ms, sequenceid=148, compaction requested=false 2024-12-03T17:16:58,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:16:58,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on f5a2a3f1964bd8ecfa7908952eaec708 2024-12-03T17:16:58,028 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f5a2a3f1964bd8ecfa7908952eaec708 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T17:16:58,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=A 2024-12-03T17:16:58,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:58,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=B 2024-12-03T17:16:58,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:58,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=C 2024-12-03T17:16:58,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:58,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/2fb94978b2f54b278d2753fcae9ca9de is 50, key is test_row_0/A:col10/1733246216868/Put/seqid=0 2024-12-03T17:16:58,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742046_1222 (size=14541) 2024-12-03T17:16:58,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:58,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42132 deadline: 1733246278047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:58,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:58,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:58,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:58,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-03T17:16:58,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:16:58,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. as already flushing 2024-12-03T17:16:58,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:16:58,065 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:58,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:58,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:58,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:58,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246278049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:58,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/2fb94978b2f54b278d2753fcae9ca9de 2024-12-03T17:16:58,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:58,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:58,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:58,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:58,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246278063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:58,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:58,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246278067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:58,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:58,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246278068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:58,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:58,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:58,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:58,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:58,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:58,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:58,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:58,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:58,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:58,081 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/a5ef0e9a526d4faf8fe1387106609417 is 50, key is test_row_0/B:col10/1733246216868/Put/seqid=0 2024-12-03T17:16:58,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742047_1223 (size=12151) 2024-12-03T17:16:58,120 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/a5ef0e9a526d4faf8fe1387106609417 2024-12-03T17:16:58,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/bbfa6e1bf9d64a409ee3a291a54be0c3 is 50, key is test_row_0/C:col10/1733246216868/Put/seqid=0 2024-12-03T17:16:58,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:58,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42132 deadline: 1733246278176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:58,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:58,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:58,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:58,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:58,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246278176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:58,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:58,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:58,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:58,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:58,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246278184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:58,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:58,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:58,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246278184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:58,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246278185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:58,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:58,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:58,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:58,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:58,197 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:58,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:58,197 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:58,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:58,197 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:58,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742048_1224 (size=12151) 2024-12-03T17:16:58,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-03T17:16:58,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:16:58,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. as already flushing 2024-12-03T17:16:58,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:16:58,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:58,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:58,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:58,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-03T17:16:58,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-03T17:16:58,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:16:58,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. as already flushing 2024-12-03T17:16:58,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:16:58,387 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:58,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:58,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:58,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42132 deadline: 1733246278385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:58,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:58,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:58,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:58,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:58,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:58,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246278395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:58,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:58,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:58,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:58,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:58,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246278407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:58,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:58,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:58,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:58,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246278411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:58,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:58,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:58,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:58,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:58,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:58,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246278411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:58,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:58,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:58,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:58,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-03T17:16:58,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:16:58,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. as already flushing 2024-12-03T17:16:58,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:16:58,554 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:58,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:58,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:58,623 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/bbfa6e1bf9d64a409ee3a291a54be0c3 2024-12-03T17:16:58,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/2fb94978b2f54b278d2753fcae9ca9de as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/2fb94978b2f54b278d2753fcae9ca9de 2024-12-03T17:16:58,704 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/2fb94978b2f54b278d2753fcae9ca9de, entries=200, sequenceid=169, filesize=14.2 K 2024-12-03T17:16:58,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:58,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246278704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:58,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/a5ef0e9a526d4faf8fe1387106609417 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/a5ef0e9a526d4faf8fe1387106609417 2024-12-03T17:16:58,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-03T17:16:58,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:16:58,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. as already flushing 2024-12-03T17:16:58,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:16:58,724 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:58,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:58,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:58,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42132 deadline: 1733246278705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:58,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:16:58,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:58,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:58,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:58,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:58,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246278724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:58,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:58,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246278724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:58,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:58,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:58,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:58,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:58,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:58,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:58,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:58,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:58,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:58,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:58,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246278724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:58,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:58,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:58,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:58,746 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/a5ef0e9a526d4faf8fe1387106609417, entries=150, sequenceid=169, filesize=11.9 K 2024-12-03T17:16:58,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/bbfa6e1bf9d64a409ee3a291a54be0c3 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/bbfa6e1bf9d64a409ee3a291a54be0c3 2024-12-03T17:16:58,795 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/bbfa6e1bf9d64a409ee3a291a54be0c3, entries=150, sequenceid=169, filesize=11.9 K 2024-12-03T17:16:58,798 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=73.80 KB/75570 for f5a2a3f1964bd8ecfa7908952eaec708 in 770ms, sequenceid=169, compaction requested=true 2024-12-03T17:16:58,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:16:58,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5a2a3f1964bd8ecfa7908952eaec708:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T17:16:58,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:58,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5a2a3f1964bd8ecfa7908952eaec708:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T17:16:58,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:16:58,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5a2a3f1964bd8ecfa7908952eaec708:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T17:16:58,798 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:16:58,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T17:16:58,799 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:16:58,803 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:16:58,803 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): f5a2a3f1964bd8ecfa7908952eaec708/B is initiating minor compaction (all files) 2024-12-03T17:16:58,804 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5a2a3f1964bd8ecfa7908952eaec708/B in TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:16:58,804 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/12b7b6fd1ea3404a85e80c7fb8f34bb5, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/f079c3008ae84b72875e236b04e02148, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/a5ef0e9a526d4faf8fe1387106609417] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp, totalSize=35.8 K 2024-12-03T17:16:58,804 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39033 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:16:58,805 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): f5a2a3f1964bd8ecfa7908952eaec708/A is initiating minor compaction (all files) 2024-12-03T17:16:58,805 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5a2a3f1964bd8ecfa7908952eaec708/A in TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:16:58,805 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/ad1c140c58e74febb55311230e64e66a, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/0b77a685fb7a423d8903655a55e3a13c, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/2fb94978b2f54b278d2753fcae9ca9de] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp, totalSize=38.1 K 2024-12-03T17:16:58,805 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12b7b6fd1ea3404a85e80c7fb8f34bb5, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733246215215 2024-12-03T17:16:58,806 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting ad1c140c58e74febb55311230e64e66a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733246215215 2024-12-03T17:16:58,806 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting f079c3008ae84b72875e236b04e02148, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1733246216448 2024-12-03T17:16:58,806 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b77a685fb7a423d8903655a55e3a13c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1733246216448 2024-12-03T17:16:58,807 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5ef0e9a526d4faf8fe1387106609417, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733246216846 2024-12-03T17:16:58,807 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fb94978b2f54b278d2753fcae9ca9de, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733246216846 2024-12-03T17:16:58,829 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5a2a3f1964bd8ecfa7908952eaec708#B#compaction#183 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:16:58,831 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/a4e7622e443b41e980007297754eb32b is 50, key is test_row_0/B:col10/1733246216868/Put/seqid=0 2024-12-03T17:16:58,834 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5a2a3f1964bd8ecfa7908952eaec708#A#compaction#184 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:16:58,835 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/81fec17dff884914b2b8a19f7f46caf3 is 50, key is test_row_0/A:col10/1733246216868/Put/seqid=0 2024-12-03T17:16:58,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742049_1225 (size=12493) 2024-12-03T17:16:58,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-03T17:16:58,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:16:58,887 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing f5a2a3f1964bd8ecfa7908952eaec708 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-03T17:16:58,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=A 2024-12-03T17:16:58,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:58,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=B 2024-12-03T17:16:58,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:58,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=C 2024-12-03T17:16:58,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:16:58,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742050_1226 (size=12493) 2024-12-03T17:16:58,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/a8eeee1d804443ca81b0646500b95d84 is 50, key is test_row_0/A:col10/1733246218057/Put/seqid=0 2024-12-03T17:16:58,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742051_1227 (size=12151) 2024-12-03T17:16:58,950 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/a8eeee1d804443ca81b0646500b95d84 2024-12-03T17:16:58,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/885e5819c0814da7bd72b68d4be9f458 is 50, key is test_row_0/B:col10/1733246218057/Put/seqid=0 2024-12-03T17:16:59,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742052_1228 (size=12151) 2024-12-03T17:16:59,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on f5a2a3f1964bd8ecfa7908952eaec708 2024-12-03T17:16:59,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. as already flushing 2024-12-03T17:16:59,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-03T17:16:59,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:59,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246279260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:59,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:59,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:59,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246279262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:59,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:59,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:59,267 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:59,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:59,267 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:59,267 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:59,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246279263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:59,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:59,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:59,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:59,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:59,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246279264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:59,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:59,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42132 deadline: 1733246279264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:59,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:59,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:59,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:59,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:59,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:59,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:59,297 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/a4e7622e443b41e980007297754eb32b as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/a4e7622e443b41e980007297754eb32b 2024-12-03T17:16:59,304 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5a2a3f1964bd8ecfa7908952eaec708/B of f5a2a3f1964bd8ecfa7908952eaec708 into a4e7622e443b41e980007297754eb32b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:16:59,304 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:16:59,304 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., storeName=f5a2a3f1964bd8ecfa7908952eaec708/B, priority=13, startTime=1733246218798; duration=0sec 2024-12-03T17:16:59,304 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:16:59,304 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5a2a3f1964bd8ecfa7908952eaec708:B 2024-12-03T17:16:59,304 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:16:59,305 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:16:59,306 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): f5a2a3f1964bd8ecfa7908952eaec708/C is initiating minor compaction (all files) 2024-12-03T17:16:59,306 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5a2a3f1964bd8ecfa7908952eaec708/C in TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:16:59,306 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/6977b13fc6b34ff8960037134802ac29, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/0bba7d0abee84966b20619340a2a3c93, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/bbfa6e1bf9d64a409ee3a291a54be0c3] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp, totalSize=35.8 K 2024-12-03T17:16:59,306 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6977b13fc6b34ff8960037134802ac29, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1733246215215 2024-12-03T17:16:59,307 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bba7d0abee84966b20619340a2a3c93, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1733246216448 2024-12-03T17:16:59,307 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbfa6e1bf9d64a409ee3a291a54be0c3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733246216846 2024-12-03T17:16:59,312 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/81fec17dff884914b2b8a19f7f46caf3 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/81fec17dff884914b2b8a19f7f46caf3 2024-12-03T17:16:59,320 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5a2a3f1964bd8ecfa7908952eaec708/A of f5a2a3f1964bd8ecfa7908952eaec708 into 81fec17dff884914b2b8a19f7f46caf3(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:16:59,320 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:16:59,320 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., storeName=f5a2a3f1964bd8ecfa7908952eaec708/A, priority=13, startTime=1733246218798; duration=0sec 2024-12-03T17:16:59,320 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:59,320 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5a2a3f1964bd8ecfa7908952eaec708:A 2024-12-03T17:16:59,325 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5a2a3f1964bd8ecfa7908952eaec708#C#compaction#187 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:16:59,326 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/766711724f5b4b58a2293e249250778b is 50, key is test_row_0/C:col10/1733246216868/Put/seqid=0 2024-12-03T17:16:59,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742053_1229 (size=12493) 2024-12-03T17:16:59,364 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/766711724f5b4b58a2293e249250778b as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/766711724f5b4b58a2293e249250778b 2024-12-03T17:16:59,372 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5a2a3f1964bd8ecfa7908952eaec708/C of f5a2a3f1964bd8ecfa7908952eaec708 into 766711724f5b4b58a2293e249250778b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:16:59,372 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:16:59,372 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., storeName=f5a2a3f1964bd8ecfa7908952eaec708/C, priority=13, startTime=1733246218798; duration=0sec 2024-12-03T17:16:59,372 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:16:59,372 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5a2a3f1964bd8ecfa7908952eaec708:C 2024-12-03T17:16:59,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:59,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246279373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:59,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:59,377 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:59,377 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:59,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246279374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:59,377 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:59,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:59,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:59,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246279374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:59,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:59,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:59,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:59,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:59,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:59,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:59,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246279374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:59,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:59,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:59,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:59,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:59,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42132 deadline: 1733246279393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:59,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:59,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:59,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:59,407 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/885e5819c0814da7bd72b68d4be9f458 2024-12-03T17:16:59,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/3fd05673394c4cb899ad99c84fe07bc0 is 50, key is test_row_0/C:col10/1733246218057/Put/seqid=0 2024-12-03T17:16:59,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742054_1230 (size=12151) 2024-12-03T17:16:59,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:59,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246279585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:59,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:59,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246279585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:59,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:59,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:59,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:59,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:59,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:59,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:59,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:59,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246279585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:59,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:59,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246279586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:59,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:59,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:59,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:59,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:59,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:59,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:59,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:59,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42132 deadline: 1733246279605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:59,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:59,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:59,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:59,835 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/3fd05673394c4cb899ad99c84fe07bc0 2024-12-03T17:16:59,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/a8eeee1d804443ca81b0646500b95d84 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/a8eeee1d804443ca81b0646500b95d84 2024-12-03T17:16:59,851 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/a8eeee1d804443ca81b0646500b95d84, entries=150, sequenceid=183, filesize=11.9 K 2024-12-03T17:16:59,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/885e5819c0814da7bd72b68d4be9f458 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/885e5819c0814da7bd72b68d4be9f458 2024-12-03T17:16:59,858 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/885e5819c0814da7bd72b68d4be9f458, entries=150, sequenceid=183, filesize=11.9 K 2024-12-03T17:16:59,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/3fd05673394c4cb899ad99c84fe07bc0 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/3fd05673394c4cb899ad99c84fe07bc0 2024-12-03T17:16:59,867 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/3fd05673394c4cb899ad99c84fe07bc0, entries=150, sequenceid=183, filesize=11.9 K 2024-12-03T17:16:59,869 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for f5a2a3f1964bd8ecfa7908952eaec708 in 982ms, sequenceid=183, compaction requested=false 2024-12-03T17:16:59,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:16:59,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:16:59,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-12-03T17:16:59,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-12-03T17:16:59,874 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-12-03T17:16:59,874 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7670 sec 2024-12-03T17:16:59,878 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 2.7910 sec 2024-12-03T17:16:59,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on f5a2a3f1964bd8ecfa7908952eaec708 2024-12-03T17:16:59,910 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f5a2a3f1964bd8ecfa7908952eaec708 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-03T17:16:59,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=A 2024-12-03T17:16:59,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:59,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=B 2024-12-03T17:16:59,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:59,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=C 2024-12-03T17:16:59,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:16:59,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/61686ee92e964c1c83e01b0ef3022ad5 is 50, key is test_row_0/A:col10/1733246219907/Put/seqid=0 2024-12-03T17:16:59,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:59,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42132 deadline: 1733246279930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:59,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:59,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246279935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:59,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:59,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:59,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:59,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:59,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246279935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:59,941 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:59,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:59,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:59,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:59,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:59,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:59,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:59,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246279940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:59,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:16:59,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246279947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:16:59,952 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:59,952 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:59,952 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:59,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:59,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:16:59,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:16:59,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742055_1231 (size=12151) 2024-12-03T17:16:59,960 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/61686ee92e964c1c83e01b0ef3022ad5 2024-12-03T17:16:59,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/8f417c16d2504653ae22989da192189a is 50, key is test_row_0/B:col10/1733246219907/Put/seqid=0 2024-12-03T17:17:00,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742056_1232 (size=12151) 2024-12-03T17:17:00,010 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/8f417c16d2504653ae22989da192189a 2024-12-03T17:17:00,042 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/426e6e16a74a47e6b888670ac1cfce05 is 50, key is test_row_0/C:col10/1733246219907/Put/seqid=0 2024-12-03T17:17:00,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:00,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246280045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:00,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:00,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:00,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:00,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:00,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246280053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:00,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:00,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:00,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:00,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:00,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246280064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:00,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:00,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:00,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:00,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:00,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246280066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:00,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:00,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:00,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:00,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742057_1233 (size=12151) 2024-12-03T17:17:00,106 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/426e6e16a74a47e6b888670ac1cfce05 2024-12-03T17:17:00,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/61686ee92e964c1c83e01b0ef3022ad5 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/61686ee92e964c1c83e01b0ef3022ad5 2024-12-03T17:17:00,170 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/61686ee92e964c1c83e01b0ef3022ad5, entries=150, sequenceid=208, filesize=11.9 K 2024-12-03T17:17:00,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/8f417c16d2504653ae22989da192189a as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/8f417c16d2504653ae22989da192189a 2024-12-03T17:17:00,213 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/8f417c16d2504653ae22989da192189a, entries=150, sequenceid=208, filesize=11.9 K 2024-12-03T17:17:00,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/426e6e16a74a47e6b888670ac1cfce05 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/426e6e16a74a47e6b888670ac1cfce05 2024-12-03T17:17:00,236 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/426e6e16a74a47e6b888670ac1cfce05, entries=150, sequenceid=208, filesize=11.9 K 2024-12-03T17:17:00,241 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=46.96 KB/48090 for f5a2a3f1964bd8ecfa7908952eaec708 in 331ms, sequenceid=208, compaction requested=true 2024-12-03T17:17:00,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:17:00,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5a2a3f1964bd8ecfa7908952eaec708:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T17:17:00,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:00,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5a2a3f1964bd8ecfa7908952eaec708:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T17:17:00,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:17:00,242 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:17:00,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5a2a3f1964bd8ecfa7908952eaec708:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T17:17:00,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T17:17:00,242 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:17:00,254 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:17:00,254 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): f5a2a3f1964bd8ecfa7908952eaec708/A is initiating minor compaction (all files) 2024-12-03T17:17:00,254 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5a2a3f1964bd8ecfa7908952eaec708/A in TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:00,254 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/81fec17dff884914b2b8a19f7f46caf3, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/a8eeee1d804443ca81b0646500b95d84, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/61686ee92e964c1c83e01b0ef3022ad5] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp, totalSize=35.9 K 2024-12-03T17:17:00,257 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:17:00,257 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): f5a2a3f1964bd8ecfa7908952eaec708/B is initiating minor compaction (all files) 2024-12-03T17:17:00,257 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5a2a3f1964bd8ecfa7908952eaec708/B in TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:00,257 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/a4e7622e443b41e980007297754eb32b, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/885e5819c0814da7bd72b68d4be9f458, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/8f417c16d2504653ae22989da192189a] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp, totalSize=35.9 K 2024-12-03T17:17:00,258 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81fec17dff884914b2b8a19f7f46caf3, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733246216846 2024-12-03T17:17:00,262 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting a4e7622e443b41e980007297754eb32b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733246216846 2024-12-03T17:17:00,264 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8eeee1d804443ca81b0646500b95d84, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1733246218046 2024-12-03T17:17:00,266 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 885e5819c0814da7bd72b68d4be9f458, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1733246218046 2024-12-03T17:17:00,267 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61686ee92e964c1c83e01b0ef3022ad5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733246219260 2024-12-03T17:17:00,267 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f5a2a3f1964bd8ecfa7908952eaec708 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-03T17:17:00,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=A 2024-12-03T17:17:00,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:00,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=B 2024-12-03T17:17:00,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:00,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=C 2024-12-03T17:17:00,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:00,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on f5a2a3f1964bd8ecfa7908952eaec708 2024-12-03T17:17:00,271 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f417c16d2504653ae22989da192189a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733246219260 2024-12-03T17:17:00,284 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/ee22dd4b461a419b897c6badbe1748e9 is 50, key is test_row_0/A:col10/1733246220262/Put/seqid=0 2024-12-03T17:17:00,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742058_1234 (size=12151) 2024-12-03T17:17:00,342 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/ee22dd4b461a419b897c6badbe1748e9 2024-12-03T17:17:00,345 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5a2a3f1964bd8ecfa7908952eaec708#A#compaction#193 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:17:00,346 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/b9ebb1d73d014b028170a371f764a51a is 50, key is test_row_0/A:col10/1733246219907/Put/seqid=0 2024-12-03T17:17:00,348 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5a2a3f1964bd8ecfa7908952eaec708#B#compaction#194 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:17:00,349 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/78dc38a468de48db9f7acdaf560638ed is 50, key is test_row_0/B:col10/1733246219907/Put/seqid=0 2024-12-03T17:17:00,373 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733246220332, max timestamp=1733246220361Num uniques 859; 2024-12-03T17:17:00,373 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T17:17:00,375 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733246220332, max timestamp=1733246220361Num uniques 859; 2024-12-03T17:17:00,376 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T17:17:00,382 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733246220332, max timestamp=1733246220361Num uniques 859; 2024-12-03T17:17:00,382 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T17:17:00,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742059_1235 (size=12595) 2024-12-03T17:17:00,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742060_1236 (size=12595) 2024-12-03T17:17:00,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/1cf45fad67184f99acd1fc26bb6210e9 is 50, key is test_row_0/B:col10/1733246220262/Put/seqid=0 2024-12-03T17:17:00,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:00,447 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/78dc38a468de48db9f7acdaf560638ed as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/78dc38a468de48db9f7acdaf560638ed 2024-12-03T17:17:00,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246280432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:00,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:00,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:00,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:00,455 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5a2a3f1964bd8ecfa7908952eaec708/B of f5a2a3f1964bd8ecfa7908952eaec708 into 78dc38a468de48db9f7acdaf560638ed(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:17:00,455 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:17:00,455 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., storeName=f5a2a3f1964bd8ecfa7908952eaec708/B, priority=13, startTime=1733246220242; duration=0sec 2024-12-03T17:17:00,455 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:17:00,455 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5a2a3f1964bd8ecfa7908952eaec708:B 2024-12-03T17:17:00,455 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:17:00,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742061_1237 (size=12151) 2024-12-03T17:17:00,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:00,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246280442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:00,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:00,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:00,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:00,458 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:17:00,458 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): f5a2a3f1964bd8ecfa7908952eaec708/C is initiating minor compaction (all files) 2024-12-03T17:17:00,458 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5a2a3f1964bd8ecfa7908952eaec708/C in TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:00,459 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/766711724f5b4b58a2293e249250778b, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/3fd05673394c4cb899ad99c84fe07bc0, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/426e6e16a74a47e6b888670ac1cfce05] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp, totalSize=35.9 K 2024-12-03T17:17:00,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:00,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42132 deadline: 1733246280447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:00,459 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 766711724f5b4b58a2293e249250778b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733246216846 2024-12-03T17:17:00,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:00,459 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fd05673394c4cb899ad99c84fe07bc0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1733246218046 2024-12-03T17:17:00,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246280447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:00,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:00,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:00,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:00,466 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 426e6e16a74a47e6b888670ac1cfce05, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733246219260 2024-12-03T17:17:00,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:00,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:00,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:00,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:00,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246280449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:00,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:00,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:00,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:00,525 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5a2a3f1964bd8ecfa7908952eaec708#C#compaction#196 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:17:00,526 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/525d943a7415472fabfe4d4ef36eaf02 is 50, key is test_row_0/C:col10/1733246219907/Put/seqid=0 2024-12-03T17:17:00,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:00,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742062_1238 (size=12595) 2024-12-03T17:17:00,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246280555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:00,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:00,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:00,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:00,581 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/525d943a7415472fabfe4d4ef36eaf02 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/525d943a7415472fabfe4d4ef36eaf02 2024-12-03T17:17:00,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:00,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246280564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:00,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:00,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:00,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:00,591 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5a2a3f1964bd8ecfa7908952eaec708/C of f5a2a3f1964bd8ecfa7908952eaec708 into 525d943a7415472fabfe4d4ef36eaf02(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:17:00,591 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:17:00,591 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., storeName=f5a2a3f1964bd8ecfa7908952eaec708/C, priority=13, startTime=1733246220242; duration=0sec 2024-12-03T17:17:00,591 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:00,591 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5a2a3f1964bd8ecfa7908952eaec708:C 2024-12-03T17:17:00,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:00,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246280574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:00,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:00,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:00,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:00,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:00,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246280574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:00,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:00,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:00,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:00,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:00,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246280775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:00,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:00,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:00,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:00,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:00,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246280798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:00,807 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:00,807 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:00,807 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:00,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:00,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246280803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:00,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:00,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:00,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:00,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:00,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246280806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:00,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:00,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:00,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:00,836 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/b9ebb1d73d014b028170a371f764a51a as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/b9ebb1d73d014b028170a371f764a51a 2024-12-03T17:17:00,843 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5a2a3f1964bd8ecfa7908952eaec708/A of f5a2a3f1964bd8ecfa7908952eaec708 into b9ebb1d73d014b028170a371f764a51a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:17:00,843 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:17:00,843 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., storeName=f5a2a3f1964bd8ecfa7908952eaec708/A, priority=13, startTime=1733246220242; duration=0sec 2024-12-03T17:17:00,844 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:00,844 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5a2a3f1964bd8ecfa7908952eaec708:A 2024-12-03T17:17:00,861 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/1cf45fad67184f99acd1fc26bb6210e9 2024-12-03T17:17:00,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/19a6dfe130924dbb922359df246161fa is 50, key is test_row_0/C:col10/1733246220262/Put/seqid=0 2024-12-03T17:17:00,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742063_1239 (size=12151) 2024-12-03T17:17:01,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:01,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246281093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:01,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:01,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:01,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:01,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:01,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246281113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:01,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:01,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:01,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:01,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:01,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246281113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:01,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:01,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:01,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:01,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:01,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246281115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:01,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:01,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:01,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:01,122 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-03T17:17:01,122 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-03T17:17:01,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-03T17:17:01,266 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:17:01,268 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T17:17:01,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-12-03T17:17:01,271 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T17:17:01,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-03T17:17:01,272 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T17:17:01,272 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T17:17:01,354 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/19a6dfe130924dbb922359df246161fa 2024-12-03T17:17:01,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/ee22dd4b461a419b897c6badbe1748e9 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/ee22dd4b461a419b897c6badbe1748e9 2024-12-03T17:17:01,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-03T17:17:01,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/ee22dd4b461a419b897c6badbe1748e9, entries=150, sequenceid=219, filesize=11.9 K 2024-12-03T17:17:01,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/1cf45fad67184f99acd1fc26bb6210e9 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/1cf45fad67184f99acd1fc26bb6210e9 2024-12-03T17:17:01,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-03T17:17:01,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:01,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. as already flushing 2024-12-03T17:17:01,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:01,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:01,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:01,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:01,440 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/1cf45fad67184f99acd1fc26bb6210e9, entries=150, sequenceid=219, filesize=11.9 K 2024-12-03T17:17:01,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/19a6dfe130924dbb922359df246161fa as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/19a6dfe130924dbb922359df246161fa 2024-12-03T17:17:01,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:01,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42132 deadline: 1733246281479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:01,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/19a6dfe130924dbb922359df246161fa, entries=150, sequenceid=219, filesize=11.9 K 2024-12-03T17:17:01,487 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 27e0f852a164,41167,1733246151593 for 'test_row_1' in f5a2a3f1964bd8ecfa7908952eaec708 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T17:17:01,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:01,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:01,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:01,498 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for f5a2a3f1964bd8ecfa7908952eaec708 in 1231ms, sequenceid=219, compaction requested=false 2024-12-03T17:17:01,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:17:01,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-03T17:17:01,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-03T17:17:01,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:01,595 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing f5a2a3f1964bd8ecfa7908952eaec708 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-03T17:17:01,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=A 2024-12-03T17:17:01,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:17:01,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=B 2024-12-03T17:17:01,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:17:01,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=C 2024-12-03T17:17:01,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:17:01,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on f5a2a3f1964bd8ecfa7908952eaec708 2024-12-03T17:17:01,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. as already flushing 2024-12-03T17:17:01,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/e78e8de45a984e10932d91409c5cfa6f is 50, key is test_row_0/A:col10/1733246220435/Put/seqid=0 2024-12-03T17:17:01,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742064_1240 (size=12151) 2024-12-03T17:17:01,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:01,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246281641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:01,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:01,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:01,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:01,651 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/e78e8de45a984e10932d91409c5cfa6f 2024-12-03T17:17:01,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:01,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246281644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:01,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:01,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:01,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:01,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:01,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246281646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:01,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:01,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:01,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:01,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:01,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246281646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:01,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:01,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:01,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:01,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/5f28ad23023b4f0db8b606be5d4c5009 is 50, key is test_row_0/B:col10/1733246220435/Put/seqid=0 2024-12-03T17:17:01,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742065_1241 (size=12151) 2024-12-03T17:17:01,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:01,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246281754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:01,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:01,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:01,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:01,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:01,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246281763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:01,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:01,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246281763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:01,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:01,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:01,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:01,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:01,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:01,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:01,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:01,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246281774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:01,777 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:01,777 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:01,777 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:01,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-03T17:17:01,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:01,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246281965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:01,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:01,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:01,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:01,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:01,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246281977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:01,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:01,982 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:01,982 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:01,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246281977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:01,982 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:01,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:01,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:01,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:01,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:01,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246281990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:01,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:01,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:01,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:02,108 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/5f28ad23023b4f0db8b606be5d4c5009 2024-12-03T17:17:02,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/5e14e0b166ed433a8df2b3aa9ce46136 is 50, key is test_row_0/C:col10/1733246220435/Put/seqid=0 2024-12-03T17:17:02,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742066_1242 (size=12151) 2024-12-03T17:17:02,229 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/5e14e0b166ed433a8df2b3aa9ce46136 2024-12-03T17:17:02,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/e78e8de45a984e10932d91409c5cfa6f as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/e78e8de45a984e10932d91409c5cfa6f 2024-12-03T17:17:02,274 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/e78e8de45a984e10932d91409c5cfa6f, entries=150, sequenceid=258, filesize=11.9 K 2024-12-03T17:17:02,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/5f28ad23023b4f0db8b606be5d4c5009 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/5f28ad23023b4f0db8b606be5d4c5009 2024-12-03T17:17:02,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:02,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246282286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:02,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:02,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:02,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:02,307 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/5f28ad23023b4f0db8b606be5d4c5009, entries=150, sequenceid=258, filesize=11.9 K 2024-12-03T17:17:02,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:02,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246282298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:02,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:02,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246282298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:02,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:02,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:02,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:02,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:02,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246282304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:02,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:02,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:02,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:02,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:02,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:02,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:02,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/5e14e0b166ed433a8df2b3aa9ce46136 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/5e14e0b166ed433a8df2b3aa9ce46136 2024-12-03T17:17:02,321 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/5e14e0b166ed433a8df2b3aa9ce46136, entries=150, sequenceid=258, filesize=11.9 K 2024-12-03T17:17:02,322 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for f5a2a3f1964bd8ecfa7908952eaec708 in 727ms, sequenceid=258, compaction requested=true 2024-12-03T17:17:02,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:17:02,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:02,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-12-03T17:17:02,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-12-03T17:17:02,362 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-12-03T17:17:02,363 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0660 sec 2024-12-03T17:17:02,379 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 1.1000 sec 2024-12-03T17:17:02,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-03T17:17:02,415 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:17:02,417 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T17:17:02,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-12-03T17:17:02,423 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T17:17:02,424 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T17:17:02,424 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T17:17:02,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-03T17:17:02,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-03T17:17:02,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-03T17:17:02,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:02,579 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing f5a2a3f1964bd8ecfa7908952eaec708 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T17:17:02,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=A 2024-12-03T17:17:02,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:02,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=B 2024-12-03T17:17:02,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:02,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=C 2024-12-03T17:17:02,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:02,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/8003769579f0499f8d2f2dd2db288285 is 50, key is test_row_1/A:col10/1733246221644/Put/seqid=0 2024-12-03T17:17:02,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742067_1243 (size=9857) 2024-12-03T17:17:02,611 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/8003769579f0499f8d2f2dd2db288285 2024-12-03T17:17:02,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/f376bfaec8d940f6ab9c445fd30313e3 is 50, key is test_row_1/B:col10/1733246221644/Put/seqid=0 2024-12-03T17:17:02,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742068_1244 (size=9857) 2024-12-03T17:17:02,656 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/f376bfaec8d940f6ab9c445fd30313e3 2024-12-03T17:17:02,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/330e54952bd94ed58b074aa10926d2fa is 50, key is test_row_1/C:col10/1733246221644/Put/seqid=0 2024-12-03T17:17:02,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742069_1245 (size=9857) 2024-12-03T17:17:02,675 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/330e54952bd94ed58b074aa10926d2fa 2024-12-03T17:17:02,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/8003769579f0499f8d2f2dd2db288285 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/8003769579f0499f8d2f2dd2db288285 2024-12-03T17:17:02,695 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/8003769579f0499f8d2f2dd2db288285, entries=100, sequenceid=268, filesize=9.6 K 2024-12-03T17:17:02,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/f376bfaec8d940f6ab9c445fd30313e3 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/f376bfaec8d940f6ab9c445fd30313e3 2024-12-03T17:17:02,710 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/f376bfaec8d940f6ab9c445fd30313e3, entries=100, sequenceid=268, filesize=9.6 K 2024-12-03T17:17:02,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/330e54952bd94ed58b074aa10926d2fa as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/330e54952bd94ed58b074aa10926d2fa 2024-12-03T17:17:02,721 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/330e54952bd94ed58b074aa10926d2fa, entries=100, sequenceid=268, filesize=9.6 K 2024-12-03T17:17:02,724 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=0 B/0 for f5a2a3f1964bd8ecfa7908952eaec708 in 145ms, sequenceid=268, compaction requested=true 2024-12-03T17:17:02,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:17:02,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:02,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-12-03T17:17:02,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-12-03T17:17:02,734 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-12-03T17:17:02,734 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 307 msec 2024-12-03T17:17:02,737 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 318 msec 2024-12-03T17:17:02,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-03T17:17:02,746 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:17:02,751 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T17:17:02,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-12-03T17:17:02,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-03T17:17:02,770 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T17:17:02,779 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T17:17:02,779 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T17:17:02,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on f5a2a3f1964bd8ecfa7908952eaec708 2024-12-03T17:17:02,855 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f5a2a3f1964bd8ecfa7908952eaec708 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-03T17:17:02,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=A 2024-12-03T17:17:02,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:02,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=B 2024-12-03T17:17:02,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:02,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=C 2024-12-03T17:17:02,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:02,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-03T17:17:02,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/47c3f81fa98e41cabd0957726e82a6c1 is 50, key is test_row_0/A:col10/1733246222844/Put/seqid=0 2024-12-03T17:17:02,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742070_1246 (size=12301) 2024-12-03T17:17:02,926 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/47c3f81fa98e41cabd0957726e82a6c1 2024-12-03T17:17:02,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-03T17:17:02,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:02,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. as already flushing 2024-12-03T17:17:02,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:02,933 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:02,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:02,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:02,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:02,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246282919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:02,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:02,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:02,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:02,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:02,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246282930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:02,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:02,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246282936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:02,959 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:02,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:02,959 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:02,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:02,959 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:02,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:02,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:02,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246282940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:02,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:02,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:02,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:02,970 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/d8e60dde9a534a08bb5c188a685c7a9f is 50, key is test_row_0/B:col10/1733246222844/Put/seqid=0 2024-12-03T17:17:03,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742071_1247 (size=12301) 2024-12-03T17:17:03,024 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/d8e60dde9a534a08bb5c188a685c7a9f 2024-12-03T17:17:03,053 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/1f330c21ce5445f4a02ecdf5e2ffcc2c is 50, key is test_row_0/C:col10/1733246222844/Put/seqid=0 2024-12-03T17:17:03,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:03,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246283055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:03,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:03,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:03,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:03,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:03,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246283067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:03,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:03,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:03,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:03,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:03,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246283069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:03,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:03,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:03,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:03,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:03,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246283070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:03,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:03,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:03,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:03,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-03T17:17:03,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742072_1248 (size=12301) 2024-12-03T17:17:03,092 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/1f330c21ce5445f4a02ecdf5e2ffcc2c 2024-12-03T17:17:03,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-03T17:17:03,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:03,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. as already flushing 2024-12-03T17:17:03,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:03,094 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:03,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:03,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:03,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/47c3f81fa98e41cabd0957726e82a6c1 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/47c3f81fa98e41cabd0957726e82a6c1 2024-12-03T17:17:03,106 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/47c3f81fa98e41cabd0957726e82a6c1, entries=150, sequenceid=280, filesize=12.0 K 2024-12-03T17:17:03,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/d8e60dde9a534a08bb5c188a685c7a9f as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/d8e60dde9a534a08bb5c188a685c7a9f 2024-12-03T17:17:03,121 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/d8e60dde9a534a08bb5c188a685c7a9f, entries=150, sequenceid=280, filesize=12.0 K 2024-12-03T17:17:03,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/1f330c21ce5445f4a02ecdf5e2ffcc2c as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/1f330c21ce5445f4a02ecdf5e2ffcc2c 2024-12-03T17:17:03,133 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/1f330c21ce5445f4a02ecdf5e2ffcc2c, entries=150, sequenceid=280, filesize=12.0 K 2024-12-03T17:17:03,134 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for f5a2a3f1964bd8ecfa7908952eaec708 in 279ms, sequenceid=280, compaction requested=true 2024-12-03T17:17:03,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:17:03,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5a2a3f1964bd8ecfa7908952eaec708:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T17:17:03,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:17:03,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5a2a3f1964bd8ecfa7908952eaec708:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T17:17:03,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T17:17:03,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5a2a3f1964bd8ecfa7908952eaec708:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T17:17:03,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-03T17:17:03,135 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-03T17:17:03,137 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-03T17:17:03,148 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 59055 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-03T17:17:03,148 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): f5a2a3f1964bd8ecfa7908952eaec708/A is initiating minor compaction (all files) 2024-12-03T17:17:03,148 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5a2a3f1964bd8ecfa7908952eaec708/A in TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:03,148 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/b9ebb1d73d014b028170a371f764a51a, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/ee22dd4b461a419b897c6badbe1748e9, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/e78e8de45a984e10932d91409c5cfa6f, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/8003769579f0499f8d2f2dd2db288285, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/47c3f81fa98e41cabd0957726e82a6c1] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp, totalSize=57.7 K 2024-12-03T17:17:03,149 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting b9ebb1d73d014b028170a371f764a51a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733246219260 2024-12-03T17:17:03,150 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting ee22dd4b461a419b897c6badbe1748e9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1733246219932 2024-12-03T17:17:03,150 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting e78e8de45a984e10932d91409c5cfa6f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1733246220435 2024-12-03T17:17:03,151 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 8003769579f0499f8d2f2dd2db288285, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1733246221639 2024-12-03T17:17:03,151 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 47c3f81fa98e41cabd0957726e82a6c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733246222809 2024-12-03T17:17:03,162 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 59055 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-03T17:17:03,163 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): f5a2a3f1964bd8ecfa7908952eaec708/C is initiating minor compaction (all files) 2024-12-03T17:17:03,163 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5a2a3f1964bd8ecfa7908952eaec708/C in TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:03,163 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/525d943a7415472fabfe4d4ef36eaf02, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/19a6dfe130924dbb922359df246161fa, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/5e14e0b166ed433a8df2b3aa9ce46136, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/330e54952bd94ed58b074aa10926d2fa, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/1f330c21ce5445f4a02ecdf5e2ffcc2c] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp, totalSize=57.7 K 2024-12-03T17:17:03,165 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5a2a3f1964bd8ecfa7908952eaec708#A#compaction#207 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:17:03,165 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/ad2f8b21e9d94f288d248b0501b06f91 is 50, key is test_row_0/A:col10/1733246222844/Put/seqid=0 2024-12-03T17:17:03,166 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 525d943a7415472fabfe4d4ef36eaf02, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733246219260 2024-12-03T17:17:03,170 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19a6dfe130924dbb922359df246161fa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1733246219932 2024-12-03T17:17:03,178 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e14e0b166ed433a8df2b3aa9ce46136, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1733246220435 2024-12-03T17:17:03,180 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 330e54952bd94ed58b074aa10926d2fa, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1733246221639 2024-12-03T17:17:03,180 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f330c21ce5445f4a02ecdf5e2ffcc2c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733246222809 2024-12-03T17:17:03,206 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5a2a3f1964bd8ecfa7908952eaec708#C#compaction#208 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:17:03,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742073_1249 (size=12915) 2024-12-03T17:17:03,208 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/a1f8ca48266749899c6980382cc9c0eb is 50, key is test_row_0/C:col10/1733246222844/Put/seqid=0 2024-12-03T17:17:03,246 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/ad2f8b21e9d94f288d248b0501b06f91 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/ad2f8b21e9d94f288d248b0501b06f91 2024-12-03T17:17:03,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742074_1250 (size=12915) 2024-12-03T17:17:03,260 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in f5a2a3f1964bd8ecfa7908952eaec708/A of f5a2a3f1964bd8ecfa7908952eaec708 into ad2f8b21e9d94f288d248b0501b06f91(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:17:03,260 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:17:03,260 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., storeName=f5a2a3f1964bd8ecfa7908952eaec708/A, priority=11, startTime=1733246223134; duration=0sec 2024-12-03T17:17:03,260 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:17:03,260 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5a2a3f1964bd8ecfa7908952eaec708:A 2024-12-03T17:17:03,261 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-03T17:17:03,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-03T17:17:03,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:03,267 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing f5a2a3f1964bd8ecfa7908952eaec708 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T17:17:03,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=A 2024-12-03T17:17:03,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:03,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=B 2024-12-03T17:17:03,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:03,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=C 2024-12-03T17:17:03,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:03,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on f5a2a3f1964bd8ecfa7908952eaec708 2024-12-03T17:17:03,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/2d19e751a69145b8a8509ef3d5d8f0f8 is 50, key is test_row_0/A:col10/1733246222916/Put/seqid=0 2024-12-03T17:17:03,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. as already flushing 2024-12-03T17:17:03,297 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 59055 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-03T17:17:03,297 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): f5a2a3f1964bd8ecfa7908952eaec708/B is initiating minor compaction (all files) 2024-12-03T17:17:03,297 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5a2a3f1964bd8ecfa7908952eaec708/B in TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:03,297 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/78dc38a468de48db9f7acdaf560638ed, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/1cf45fad67184f99acd1fc26bb6210e9, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/5f28ad23023b4f0db8b606be5d4c5009, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/f376bfaec8d940f6ab9c445fd30313e3, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/d8e60dde9a534a08bb5c188a685c7a9f] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp, totalSize=57.7 K 2024-12-03T17:17:03,302 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 78dc38a468de48db9f7acdaf560638ed, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733246219260 2024-12-03T17:17:03,304 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cf45fad67184f99acd1fc26bb6210e9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1733246219932 2024-12-03T17:17:03,314 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f28ad23023b4f0db8b606be5d4c5009, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1733246220435 2024-12-03T17:17:03,318 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting f376bfaec8d940f6ab9c445fd30313e3, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1733246221639 2024-12-03T17:17:03,322 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting d8e60dde9a534a08bb5c188a685c7a9f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733246222809 2024-12-03T17:17:03,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742075_1251 (size=12301) 2024-12-03T17:17:03,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:03,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246283328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:03,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:03,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246283331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:03,342 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/2d19e751a69145b8a8509ef3d5d8f0f8 2024-12-03T17:17:03,343 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:03,343 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:03,343 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:03,344 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5a2a3f1964bd8ecfa7908952eaec708#B#compaction#210 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:17:03,345 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/4d2d00110cbd412ab00e377cd77379b4 is 50, key is test_row_0/B:col10/1733246222844/Put/seqid=0 2024-12-03T17:17:03,347 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:03,347 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:03,347 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:03,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:03,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246283332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:03,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:03,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:03,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:03,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:03,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246283342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:03,359 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:03,359 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:03,359 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:03,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/01c35a2eed7a42fdbbe97d4767cfdda2 is 50, key is test_row_0/B:col10/1733246222916/Put/seqid=0 2024-12-03T17:17:03,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742076_1252 (size=12915) 2024-12-03T17:17:03,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-03T17:17:03,397 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/4d2d00110cbd412ab00e377cd77379b4 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/4d2d00110cbd412ab00e377cd77379b4 2024-12-03T17:17:03,405 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in f5a2a3f1964bd8ecfa7908952eaec708/B of f5a2a3f1964bd8ecfa7908952eaec708 into 4d2d00110cbd412ab00e377cd77379b4(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:17:03,405 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:17:03,405 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., storeName=f5a2a3f1964bd8ecfa7908952eaec708/B, priority=11, startTime=1733246223135; duration=0sec 2024-12-03T17:17:03,405 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:03,405 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5a2a3f1964bd8ecfa7908952eaec708:B 2024-12-03T17:17:03,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742077_1253 (size=12301) 2024-12-03T17:17:03,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:03,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246283454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:03,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:03,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246283454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:03,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:03,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:03,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:03,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:03,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:03,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:03,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:03,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246283454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:03,459 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:03,459 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:03,459 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:03,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:03,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246283464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:03,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:03,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:03,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:03,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:03,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42132 deadline: 1733246283513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:03,518 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 27e0f852a164,41167,1733246151593 for 'test_row_1' in f5a2a3f1964bd8ecfa7908952eaec708 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4254 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T17:17:03,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:03,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:03,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:03,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:03,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246283664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:03,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:03,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:03,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:03,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:03,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246283664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:03,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:03,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246283665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:03,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:03,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:03,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:03,668 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:03,668 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:03,668 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:03,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:03,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246283679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:03,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:03,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:03,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:03,724 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/a1f8ca48266749899c6980382cc9c0eb as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/a1f8ca48266749899c6980382cc9c0eb 2024-12-03T17:17:03,753 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in f5a2a3f1964bd8ecfa7908952eaec708/C of f5a2a3f1964bd8ecfa7908952eaec708 into a1f8ca48266749899c6980382cc9c0eb(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:17:03,753 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:17:03,753 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., storeName=f5a2a3f1964bd8ecfa7908952eaec708/C, priority=11, startTime=1733246223135; duration=0sec 2024-12-03T17:17:03,753 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:03,753 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5a2a3f1964bd8ecfa7908952eaec708:C 2024-12-03T17:17:03,842 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/01c35a2eed7a42fdbbe97d4767cfdda2 2024-12-03T17:17:03,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/257537db81bf4c6bb7837758f4c57a78 is 50, key is test_row_0/C:col10/1733246222916/Put/seqid=0 2024-12-03T17:17:03,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-03T17:17:03,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742078_1254 (size=12301) 2024-12-03T17:17:03,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:03,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246283974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:03,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:03,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:03,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:03,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:03,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246283974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:03,977 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:03,977 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:03,977 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:03,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:03,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246283974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:03,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:03,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:03,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:03,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:03,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246283993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:03,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:03,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:03,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:04,324 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/257537db81bf4c6bb7837758f4c57a78 2024-12-03T17:17:04,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/2d19e751a69145b8a8509ef3d5d8f0f8 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/2d19e751a69145b8a8509ef3d5d8f0f8 2024-12-03T17:17:04,375 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/2d19e751a69145b8a8509ef3d5d8f0f8, entries=150, sequenceid=301, filesize=12.0 K 2024-12-03T17:17:04,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/01c35a2eed7a42fdbbe97d4767cfdda2 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/01c35a2eed7a42fdbbe97d4767cfdda2 2024-12-03T17:17:04,423 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/01c35a2eed7a42fdbbe97d4767cfdda2, entries=150, sequenceid=301, filesize=12.0 K 2024-12-03T17:17:04,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/257537db81bf4c6bb7837758f4c57a78 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/257537db81bf4c6bb7837758f4c57a78 2024-12-03T17:17:04,441 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/257537db81bf4c6bb7837758f4c57a78, entries=150, sequenceid=301, filesize=12.0 K 2024-12-03T17:17:04,443 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for f5a2a3f1964bd8ecfa7908952eaec708 in 1176ms, sequenceid=301, compaction requested=false 2024-12-03T17:17:04,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:17:04,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:04,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-12-03T17:17:04,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-12-03T17:17:04,476 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-12-03T17:17:04,476 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6760 sec 2024-12-03T17:17:04,506 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.7310 sec 2024-12-03T17:17:04,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on f5a2a3f1964bd8ecfa7908952eaec708 2024-12-03T17:17:04,508 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f5a2a3f1964bd8ecfa7908952eaec708 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-03T17:17:04,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=A 2024-12-03T17:17:04,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:04,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=B 2024-12-03T17:17:04,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:04,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=C 2024-12-03T17:17:04,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:04,542 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/c0ff985e679f4f58bc9b278ae4d7d3e0 is 50, key is test_row_0/A:col10/1733246223329/Put/seqid=0 2024-12-03T17:17:04,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742079_1255 (size=14741) 2024-12-03T17:17:04,586 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/c0ff985e679f4f58bc9b278ae4d7d3e0 2024-12-03T17:17:04,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:04,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246284583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:04,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:04,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:04,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:04,600 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/7b866d16484f4b78aba71a7028746b8e is 50, key is test_row_0/B:col10/1733246223329/Put/seqid=0 2024-12-03T17:17:04,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:04,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246284593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:04,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:04,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:04,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:04,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246284594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:04,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:04,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:04,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246284595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:04,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:04,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:04,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:04,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:04,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:04,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:04,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742080_1256 (size=12301) 2024-12-03T17:17:04,656 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/7b866d16484f4b78aba71a7028746b8e 2024-12-03T17:17:04,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/dc361fc2cdaf44228b64044faf4a9171 is 50, key is test_row_0/C:col10/1733246223329/Put/seqid=0 2024-12-03T17:17:04,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:04,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246284707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:04,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742081_1257 (size=12301) 2024-12-03T17:17:04,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:04,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:04,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:04,719 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/dc361fc2cdaf44228b64044faf4a9171 2024-12-03T17:17:04,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:04,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246284713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:04,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:04,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246284713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:04,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:04,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:04,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:04,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:04,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:04,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:04,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:04,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246284714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:04,726 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:04,726 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:04,726 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:04,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/c0ff985e679f4f58bc9b278ae4d7d3e0 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/c0ff985e679f4f58bc9b278ae4d7d3e0 2024-12-03T17:17:04,775 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/c0ff985e679f4f58bc9b278ae4d7d3e0, entries=200, sequenceid=319, filesize=14.4 K 2024-12-03T17:17:04,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/7b866d16484f4b78aba71a7028746b8e as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/7b866d16484f4b78aba71a7028746b8e 2024-12-03T17:17:04,793 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/7b866d16484f4b78aba71a7028746b8e, entries=150, sequenceid=319, filesize=12.0 K 2024-12-03T17:17:04,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/dc361fc2cdaf44228b64044faf4a9171 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/dc361fc2cdaf44228b64044faf4a9171 2024-12-03T17:17:04,811 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/dc361fc2cdaf44228b64044faf4a9171, entries=150, sequenceid=319, filesize=12.0 K 2024-12-03T17:17:04,812 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for f5a2a3f1964bd8ecfa7908952eaec708 in 304ms, sequenceid=319, compaction requested=true 2024-12-03T17:17:04,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:17:04,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5a2a3f1964bd8ecfa7908952eaec708:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T17:17:04,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:04,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5a2a3f1964bd8ecfa7908952eaec708:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T17:17:04,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:17:04,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5a2a3f1964bd8ecfa7908952eaec708:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T17:17:04,813 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:17:04,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T17:17:04,813 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:17:04,820 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:17:04,821 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): f5a2a3f1964bd8ecfa7908952eaec708/B is initiating minor compaction (all files) 2024-12-03T17:17:04,821 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5a2a3f1964bd8ecfa7908952eaec708/B in TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:04,821 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/4d2d00110cbd412ab00e377cd77379b4, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/01c35a2eed7a42fdbbe97d4767cfdda2, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/7b866d16484f4b78aba71a7028746b8e] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp, totalSize=36.6 K 2024-12-03T17:17:04,823 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39957 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:17:04,823 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): f5a2a3f1964bd8ecfa7908952eaec708/A is initiating minor compaction (all files) 2024-12-03T17:17:04,823 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5a2a3f1964bd8ecfa7908952eaec708/A in TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:04,823 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/ad2f8b21e9d94f288d248b0501b06f91, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/2d19e751a69145b8a8509ef3d5d8f0f8, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/c0ff985e679f4f58bc9b278ae4d7d3e0] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp, totalSize=39.0 K 2024-12-03T17:17:04,825 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d2d00110cbd412ab00e377cd77379b4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733246222809 2024-12-03T17:17:04,825 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting ad2f8b21e9d94f288d248b0501b06f91, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733246222809 2024-12-03T17:17:04,826 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d19e751a69145b8a8509ef3d5d8f0f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1733246222916 2024-12-03T17:17:04,827 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01c35a2eed7a42fdbbe97d4767cfdda2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1733246222916 2024-12-03T17:17:04,830 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b866d16484f4b78aba71a7028746b8e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1733246223329 2024-12-03T17:17:04,835 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting c0ff985e679f4f58bc9b278ae4d7d3e0, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1733246223329 2024-12-03T17:17:04,893 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5a2a3f1964bd8ecfa7908952eaec708#B#compaction#216 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:17:04,893 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/f2c9e62fb7674d2ab72a4881cf4892dc is 50, key is test_row_0/B:col10/1733246223329/Put/seqid=0 2024-12-03T17:17:04,897 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5a2a3f1964bd8ecfa7908952eaec708#A#compaction#217 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:17:04,898 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/49827d2dff7f4f9f897b0430760c0d54 is 50, key is test_row_0/A:col10/1733246223329/Put/seqid=0 2024-12-03T17:17:04,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-03T17:17:04,914 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:17:04,915 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T17:17:04,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-12-03T17:17:04,930 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f5a2a3f1964bd8ecfa7908952eaec708 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-03T17:17:04,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=A 2024-12-03T17:17:04,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:04,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=B 2024-12-03T17:17:04,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:04,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=C 2024-12-03T17:17:04,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:04,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on f5a2a3f1964bd8ecfa7908952eaec708 2024-12-03T17:17:04,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-03T17:17:04,935 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T17:17:04,943 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T17:17:04,943 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T17:17:04,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742082_1258 (size=13017) 2024-12-03T17:17:04,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742083_1259 (size=13017) 2024-12-03T17:17:04,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/6d521ab4e22f4962ae132139210d567d is 50, key is test_row_0/A:col10/1733246224591/Put/seqid=0 2024-12-03T17:17:04,981 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/49827d2dff7f4f9f897b0430760c0d54 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/49827d2dff7f4f9f897b0430760c0d54 2024-12-03T17:17:04,982 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/f2c9e62fb7674d2ab72a4881cf4892dc as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/f2c9e62fb7674d2ab72a4881cf4892dc 2024-12-03T17:17:04,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:04,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:04,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246284980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:04,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246284981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:04,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:04,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:04,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:04,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:04,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246284985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:04,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:04,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246284982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:04,994 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:04,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:04,994 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:04,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:04,994 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:04,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:04,996 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5a2a3f1964bd8ecfa7908952eaec708/A of f5a2a3f1964bd8ecfa7908952eaec708 into 49827d2dff7f4f9f897b0430760c0d54(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:17:04,996 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:17:04,997 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5a2a3f1964bd8ecfa7908952eaec708/B of f5a2a3f1964bd8ecfa7908952eaec708 into f2c9e62fb7674d2ab72a4881cf4892dc(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:17:04,997 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:17:04,997 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., storeName=f5a2a3f1964bd8ecfa7908952eaec708/B, priority=13, startTime=1733246224813; duration=0sec 2024-12-03T17:17:04,997 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:17:04,997 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5a2a3f1964bd8ecfa7908952eaec708:B 2024-12-03T17:17:04,997 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:17:04,998 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., storeName=f5a2a3f1964bd8ecfa7908952eaec708/A, priority=13, startTime=1733246224812; duration=0sec 2024-12-03T17:17:04,998 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:04,998 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5a2a3f1964bd8ecfa7908952eaec708:A 2024-12-03T17:17:04,999 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:17:04,999 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): f5a2a3f1964bd8ecfa7908952eaec708/C is initiating minor compaction (all files) 2024-12-03T17:17:04,999 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5a2a3f1964bd8ecfa7908952eaec708/C in TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:04,999 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/a1f8ca48266749899c6980382cc9c0eb, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/257537db81bf4c6bb7837758f4c57a78, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/dc361fc2cdaf44228b64044faf4a9171] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp, totalSize=36.6 K 2024-12-03T17:17:05,000 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1f8ca48266749899c6980382cc9c0eb, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733246222809 2024-12-03T17:17:05,004 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 257537db81bf4c6bb7837758f4c57a78, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1733246222916 2024-12-03T17:17:05,005 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc361fc2cdaf44228b64044faf4a9171, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1733246223329 2024-12-03T17:17:05,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:05,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:05,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:05,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742084_1260 (size=14741) 2024-12-03T17:17:05,019 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/6d521ab4e22f4962ae132139210d567d 2024-12-03T17:17:05,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-03T17:17:05,061 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5a2a3f1964bd8ecfa7908952eaec708#C#compaction#219 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:17:05,062 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/7eba619ed24f461ab0623f0b4d97bd22 is 50, key is test_row_0/C:col10/1733246223329/Put/seqid=0 2024-12-03T17:17:05,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/4182e6e291234e749dce865f1302a127 is 50, key is test_row_0/B:col10/1733246224591/Put/seqid=0 2024-12-03T17:17:05,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-03T17:17:05,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:05,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. as already flushing 2024-12-03T17:17:05,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:05,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:05,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:05,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:05,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246285107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:05,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:05,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246285107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:05,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:05,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:05,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:05,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:05,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:05,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:05,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:05,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:05,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246285108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:05,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:05,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:05,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:05,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742085_1261 (size=13017) 2024-12-03T17:17:05,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:05,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246285113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:05,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:05,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:05,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:05,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742086_1262 (size=12301) 2024-12-03T17:17:05,137 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/4182e6e291234e749dce865f1302a127 2024-12-03T17:17:05,145 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/7eba619ed24f461ab0623f0b4d97bd22 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/7eba619ed24f461ab0623f0b4d97bd22 2024-12-03T17:17:05,148 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/fb2d90e170524ff09fd0f9d1663765d4 is 50, key is test_row_0/C:col10/1733246224591/Put/seqid=0 2024-12-03T17:17:05,152 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5a2a3f1964bd8ecfa7908952eaec708/C of f5a2a3f1964bd8ecfa7908952eaec708 into 7eba619ed24f461ab0623f0b4d97bd22(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:17:05,152 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:17:05,152 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., storeName=f5a2a3f1964bd8ecfa7908952eaec708/C, priority=13, startTime=1733246224813; duration=0sec 2024-12-03T17:17:05,152 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:05,152 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5a2a3f1964bd8ecfa7908952eaec708:C 2024-12-03T17:17:05,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742087_1263 (size=12301) 2024-12-03T17:17:05,219 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/fb2d90e170524ff09fd0f9d1663765d4 2024-12-03T17:17:05,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/6d521ab4e22f4962ae132139210d567d as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/6d521ab4e22f4962ae132139210d567d 2024-12-03T17:17:05,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-03T17:17:05,254 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/6d521ab4e22f4962ae132139210d567d, entries=200, sequenceid=337, filesize=14.4 K 2024-12-03T17:17:05,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/4182e6e291234e749dce865f1302a127 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/4182e6e291234e749dce865f1302a127 2024-12-03T17:17:05,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-03T17:17:05,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:05,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. as already flushing 2024-12-03T17:17:05,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:05,275 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:05,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:05,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:05,291 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/4182e6e291234e749dce865f1302a127, entries=150, sequenceid=337, filesize=12.0 K 2024-12-03T17:17:05,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/fb2d90e170524ff09fd0f9d1663765d4 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/fb2d90e170524ff09fd0f9d1663765d4 2024-12-03T17:17:05,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:05,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246285315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:05,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:05,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246285315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:05,322 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:05,322 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:05,323 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:05,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:05,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:05,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:05,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:05,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246285326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:05,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:05,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:05,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:05,332 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/fb2d90e170524ff09fd0f9d1663765d4, entries=150, sequenceid=337, filesize=12.0 K 2024-12-03T17:17:05,333 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for f5a2a3f1964bd8ecfa7908952eaec708 in 404ms, sequenceid=337, compaction requested=false 2024-12-03T17:17:05,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:17:05,345 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f5a2a3f1964bd8ecfa7908952eaec708 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-03T17:17:05,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=A 2024-12-03T17:17:05,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:05,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=B 2024-12-03T17:17:05,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:05,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=C 2024-12-03T17:17:05,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:05,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on f5a2a3f1964bd8ecfa7908952eaec708 2024-12-03T17:17:05,352 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/76113bb8a795464a9ffa90a809c96bed is 50, key is test_row_0/A:col10/1733246224982/Put/seqid=0 2024-12-03T17:17:05,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742088_1264 (size=12301) 2024-12-03T17:17:05,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-03T17:17:05,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:05,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. as already flushing 2024-12-03T17:17:05,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:05,432 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:05,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:05,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:05,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:05,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246285494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:05,500 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:05,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:05,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:05,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-03T17:17:05,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-03T17:17:05,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:05,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. as already flushing 2024-12-03T17:17:05,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:05,588 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:05,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:05,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:05,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:05,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246285604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:05,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:05,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:05,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:05,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:05,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246285633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:05,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:05,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246285633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:05,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:05,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:05,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:05,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:05,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:05,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:05,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:05,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246285645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:05,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:05,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:05,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:05,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-03T17:17:05,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:05,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. as already flushing 2024-12-03T17:17:05,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:05,743 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:05,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:05,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:05,793 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/76113bb8a795464a9ffa90a809c96bed 2024-12-03T17:17:05,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:05,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246285814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:05,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:05,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:05,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:05,828 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/831cbafc416e4c47acdc4e9bb6ac4fc2 is 50, key is test_row_0/B:col10/1733246224982/Put/seqid=0 2024-12-03T17:17:05,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742089_1265 (size=12301) 2024-12-03T17:17:05,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-03T17:17:05,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:05,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. as already flushing 2024-12-03T17:17:05,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:05,900 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:05,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:05,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:06,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-03T17:17:06,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:06,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. as already flushing 2024-12-03T17:17:06,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:06,060 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:06,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:06,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:06,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-03T17:17:06,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:06,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246286124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:06,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:06,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:06,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:06,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:06,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246286147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:06,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:06,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246286147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:06,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:06,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:06,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:06,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:06,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:06,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:06,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:06,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246286169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:06,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:06,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:06,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:06,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-03T17:17:06,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:06,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. as already flushing 2024-12-03T17:17:06,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:06,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:06,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:06,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:06,283 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/831cbafc416e4c47acdc4e9bb6ac4fc2 2024-12-03T17:17:06,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/4e9e96a5639644d3b45f203b96dccb11 is 50, key is test_row_0/C:col10/1733246224982/Put/seqid=0 2024-12-03T17:17:06,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742090_1266 (size=12301) 2024-12-03T17:17:06,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-03T17:17:06,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:06,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. as already flushing 2024-12-03T17:17:06,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:06,398 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:06,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:06,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:06,566 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-03T17:17:06,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:06,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. as already flushing 2024-12-03T17:17:06,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:06,567 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:06,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:06,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:06,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:06,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246286634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:06,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:06,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:06,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:06,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-03T17:17:06,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:06,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. as already flushing 2024-12-03T17:17:06,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:06,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:06,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:06,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:06,810 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/4e9e96a5639644d3b45f203b96dccb11 2024-12-03T17:17:06,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/76113bb8a795464a9ffa90a809c96bed as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/76113bb8a795464a9ffa90a809c96bed 2024-12-03T17:17:06,841 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/76113bb8a795464a9ffa90a809c96bed, entries=150, sequenceid=355, filesize=12.0 K 2024-12-03T17:17:06,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/831cbafc416e4c47acdc4e9bb6ac4fc2 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/831cbafc416e4c47acdc4e9bb6ac4fc2 2024-12-03T17:17:06,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/831cbafc416e4c47acdc4e9bb6ac4fc2, entries=150, sequenceid=355, filesize=12.0 K 2024-12-03T17:17:06,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/4e9e96a5639644d3b45f203b96dccb11 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/4e9e96a5639644d3b45f203b96dccb11 2024-12-03T17:17:06,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/4e9e96a5639644d3b45f203b96dccb11, entries=150, sequenceid=355, filesize=12.0 K 2024-12-03T17:17:06,862 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for f5a2a3f1964bd8ecfa7908952eaec708 in 1517ms, sequenceid=355, compaction requested=true 2024-12-03T17:17:06,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:17:06,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5a2a3f1964bd8ecfa7908952eaec708:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T17:17:06,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:06,863 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:17:06,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5a2a3f1964bd8ecfa7908952eaec708:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T17:17:06,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:17:06,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5a2a3f1964bd8ecfa7908952eaec708:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T17:17:06,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T17:17:06,864 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:17:06,868 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:17:06,868 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): f5a2a3f1964bd8ecfa7908952eaec708/B is initiating minor compaction (all files) 2024-12-03T17:17:06,868 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5a2a3f1964bd8ecfa7908952eaec708/B in TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:06,868 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/f2c9e62fb7674d2ab72a4881cf4892dc, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/4182e6e291234e749dce865f1302a127, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/831cbafc416e4c47acdc4e9bb6ac4fc2] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp, totalSize=36.7 K 2024-12-03T17:17:06,869 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:17:06,869 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): f5a2a3f1964bd8ecfa7908952eaec708/A is initiating minor compaction (all files) 2024-12-03T17:17:06,869 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5a2a3f1964bd8ecfa7908952eaec708/A in TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:06,869 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/49827d2dff7f4f9f897b0430760c0d54, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/6d521ab4e22f4962ae132139210d567d, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/76113bb8a795464a9ffa90a809c96bed] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp, totalSize=39.1 K 2024-12-03T17:17:06,873 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2c9e62fb7674d2ab72a4881cf4892dc, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1733246223329 2024-12-03T17:17:06,875 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 49827d2dff7f4f9f897b0430760c0d54, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1733246223329 2024-12-03T17:17:06,875 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4182e6e291234e749dce865f1302a127, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1733246224591 2024-12-03T17:17:06,876 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d521ab4e22f4962ae132139210d567d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1733246224588 2024-12-03T17:17:06,876 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 831cbafc416e4c47acdc4e9bb6ac4fc2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1733246224980 2024-12-03T17:17:06,877 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 76113bb8a795464a9ffa90a809c96bed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1733246224980 2024-12-03T17:17:06,886 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-03T17:17:06,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:06,887 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing f5a2a3f1964bd8ecfa7908952eaec708 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-03T17:17:06,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=A 2024-12-03T17:17:06,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:06,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=B 2024-12-03T17:17:06,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:06,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=C 2024-12-03T17:17:06,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:06,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/067656658ae342cea905705a39facd18 is 50, key is test_row_0/A:col10/1733246225482/Put/seqid=0 2024-12-03T17:17:06,928 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5a2a3f1964bd8ecfa7908952eaec708#B#compaction#226 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:17:06,929 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/8e0255fa618443c09bc32cbc2c6b801a is 50, key is test_row_0/B:col10/1733246224982/Put/seqid=0 2024-12-03T17:17:06,937 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5a2a3f1964bd8ecfa7908952eaec708#A#compaction#227 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:17:06,938 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/ff82b4ff19034de4b7904b2d1e2b9f52 is 50, key is test_row_0/A:col10/1733246224982/Put/seqid=0 2024-12-03T17:17:06,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742091_1267 (size=12301) 2024-12-03T17:17:06,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742092_1268 (size=13119) 2024-12-03T17:17:07,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742093_1269 (size=13119) 2024-12-03T17:17:07,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-03T17:17:07,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on f5a2a3f1964bd8ecfa7908952eaec708 2024-12-03T17:17:07,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. as already flushing 2024-12-03T17:17:07,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:07,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246287216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:07,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:07,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:07,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:07,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:07,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246287216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:07,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:07,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246287216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:07,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:07,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:07,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:07,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:07,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:07,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:07,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:07,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246287334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:07,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:07,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:07,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:07,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246287334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:07,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:07,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:07,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:07,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:07,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:07,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246287334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:07,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:07,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:07,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:07,389 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/067656658ae342cea905705a39facd18 2024-12-03T17:17:07,405 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/8e0255fa618443c09bc32cbc2c6b801a as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/8e0255fa618443c09bc32cbc2c6b801a 2024-12-03T17:17:07,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/9a4dbc9e644241fb8ccf622d11240846 is 50, key is test_row_0/B:col10/1733246225482/Put/seqid=0 2024-12-03T17:17:07,413 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/ff82b4ff19034de4b7904b2d1e2b9f52 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/ff82b4ff19034de4b7904b2d1e2b9f52 2024-12-03T17:17:07,423 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5a2a3f1964bd8ecfa7908952eaec708/B of f5a2a3f1964bd8ecfa7908952eaec708 into 8e0255fa618443c09bc32cbc2c6b801a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:17:07,423 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:17:07,423 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., storeName=f5a2a3f1964bd8ecfa7908952eaec708/B, priority=13, startTime=1733246226863; duration=0sec 2024-12-03T17:17:07,423 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:17:07,423 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5a2a3f1964bd8ecfa7908952eaec708:B 2024-12-03T17:17:07,423 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:17:07,429 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:17:07,429 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): f5a2a3f1964bd8ecfa7908952eaec708/C is initiating minor compaction (all files) 2024-12-03T17:17:07,429 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5a2a3f1964bd8ecfa7908952eaec708/C in TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:07,429 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5a2a3f1964bd8ecfa7908952eaec708/A of f5a2a3f1964bd8ecfa7908952eaec708 into ff82b4ff19034de4b7904b2d1e2b9f52(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:17:07,429 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/7eba619ed24f461ab0623f0b4d97bd22, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/fb2d90e170524ff09fd0f9d1663765d4, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/4e9e96a5639644d3b45f203b96dccb11] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp, totalSize=36.7 K 2024-12-03T17:17:07,429 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:17:07,429 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., storeName=f5a2a3f1964bd8ecfa7908952eaec708/A, priority=13, startTime=1733246226863; duration=0sec 2024-12-03T17:17:07,430 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:07,430 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5a2a3f1964bd8ecfa7908952eaec708:A 2024-12-03T17:17:07,430 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7eba619ed24f461ab0623f0b4d97bd22, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1733246223329 2024-12-03T17:17:07,431 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb2d90e170524ff09fd0f9d1663765d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1733246224591 2024-12-03T17:17:07,431 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e9e96a5639644d3b45f203b96dccb11, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1733246224980 2024-12-03T17:17:07,449 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5a2a3f1964bd8ecfa7908952eaec708#C#compaction#229 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:17:07,449 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/574543a2850c4fb0bb1074c9c2fee4eb is 50, key is test_row_0/C:col10/1733246224982/Put/seqid=0 2024-12-03T17:17:07,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742094_1270 (size=12301) 2024-12-03T17:17:07,498 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/9a4dbc9e644241fb8ccf622d11240846 2024-12-03T17:17:07,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742095_1271 (size=13119) 2024-12-03T17:17:07,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/2988a9efb43f40fab35a3bdbcf488908 is 50, key is test_row_0/C:col10/1733246225482/Put/seqid=0 2024-12-03T17:17:07,537 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/574543a2850c4fb0bb1074c9c2fee4eb as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/574543a2850c4fb0bb1074c9c2fee4eb 2024-12-03T17:17:07,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:07,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42132 deadline: 1733246287535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:07,541 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 27e0f852a164,41167,1733246151593 for 'test_row_1' in f5a2a3f1964bd8ecfa7908952eaec708 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8277 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T17:17:07,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:07,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:07,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:07,546 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5a2a3f1964bd8ecfa7908952eaec708/C of f5a2a3f1964bd8ecfa7908952eaec708 into 574543a2850c4fb0bb1074c9c2fee4eb(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:17:07,546 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:17:07,546 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., storeName=f5a2a3f1964bd8ecfa7908952eaec708/C, priority=13, startTime=1733246226863; duration=0sec 2024-12-03T17:17:07,546 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:07,546 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5a2a3f1964bd8ecfa7908952eaec708:C 2024-12-03T17:17:07,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:07,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246287544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:07,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:07,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:07,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:07,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246287545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:07,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:07,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:07,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:07,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:07,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:07,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246287545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:07,550 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:07,550 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:07,550 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:07,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742096_1272 (size=12301) 2024-12-03T17:17:07,577 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/2988a9efb43f40fab35a3bdbcf488908 2024-12-03T17:17:07,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/067656658ae342cea905705a39facd18 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/067656658ae342cea905705a39facd18 2024-12-03T17:17:07,593 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/067656658ae342cea905705a39facd18, entries=150, sequenceid=372, filesize=12.0 K 2024-12-03T17:17:07,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/9a4dbc9e644241fb8ccf622d11240846 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/9a4dbc9e644241fb8ccf622d11240846 2024-12-03T17:17:07,603 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/9a4dbc9e644241fb8ccf622d11240846, entries=150, sequenceid=372, filesize=12.0 K 2024-12-03T17:17:07,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/2988a9efb43f40fab35a3bdbcf488908 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/2988a9efb43f40fab35a3bdbcf488908 2024-12-03T17:17:07,607 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/2988a9efb43f40fab35a3bdbcf488908, entries=150, sequenceid=372, filesize=12.0 K 2024-12-03T17:17:07,609 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for f5a2a3f1964bd8ecfa7908952eaec708 in 721ms, sequenceid=372, compaction requested=false 2024-12-03T17:17:07,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:17:07,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:07,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-12-03T17:17:07,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-12-03T17:17:07,620 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-12-03T17:17:07,620 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6720 sec 2024-12-03T17:17:07,626 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 2.7090 sec 2024-12-03T17:17:07,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on f5a2a3f1964bd8ecfa7908952eaec708 2024-12-03T17:17:07,649 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f5a2a3f1964bd8ecfa7908952eaec708 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-03T17:17:07,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=A 2024-12-03T17:17:07,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:07,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=B 2024-12-03T17:17:07,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:07,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=C 2024-12-03T17:17:07,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:07,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/cc73b2aac4244ebaa1d055c4d87207a3 is 50, key is test_row_0/A:col10/1733246227211/Put/seqid=0 2024-12-03T17:17:07,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742097_1273 (size=12301) 2024-12-03T17:17:07,708 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/cc73b2aac4244ebaa1d055c4d87207a3 2024-12-03T17:17:07,745 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/72a37d808998408bb4125610b33eefd0 is 50, key is test_row_0/B:col10/1733246227211/Put/seqid=0 2024-12-03T17:17:07,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:07,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742098_1274 (size=12301) 2024-12-03T17:17:07,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246287783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:07,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:07,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:07,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:07,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:07,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246287853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:07,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:07,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:07,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:07,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:07,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246287855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:07,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:07,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:07,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:07,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:07,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246287864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:07,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:07,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:07,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:07,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:07,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246287910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:07,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:07,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:07,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:08,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:08,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246288127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:08,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:08,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:08,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:08,197 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/72a37d808998408bb4125610b33eefd0 2024-12-03T17:17:08,229 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/4c630e530ea1445898703f42db9fb623 is 50, key is test_row_0/C:col10/1733246227211/Put/seqid=0 2024-12-03T17:17:08,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742099_1275 (size=12301) 2024-12-03T17:17:08,248 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/4c630e530ea1445898703f42db9fb623 2024-12-03T17:17:08,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/cc73b2aac4244ebaa1d055c4d87207a3 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/cc73b2aac4244ebaa1d055c4d87207a3 2024-12-03T17:17:08,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/cc73b2aac4244ebaa1d055c4d87207a3, entries=150, sequenceid=392, filesize=12.0 K 2024-12-03T17:17:08,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/72a37d808998408bb4125610b33eefd0 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/72a37d808998408bb4125610b33eefd0 2024-12-03T17:17:08,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/72a37d808998408bb4125610b33eefd0, entries=150, sequenceid=392, filesize=12.0 K 2024-12-03T17:17:08,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/4c630e530ea1445898703f42db9fb623 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/4c630e530ea1445898703f42db9fb623 2024-12-03T17:17:08,292 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/4c630e530ea1445898703f42db9fb623, entries=150, sequenceid=392, filesize=12.0 K 2024-12-03T17:17:08,293 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for f5a2a3f1964bd8ecfa7908952eaec708 in 644ms, sequenceid=392, compaction requested=true 2024-12-03T17:17:08,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:17:08,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5a2a3f1964bd8ecfa7908952eaec708:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T17:17:08,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:08,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5a2a3f1964bd8ecfa7908952eaec708:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T17:17:08,293 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:17:08,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:08,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5a2a3f1964bd8ecfa7908952eaec708:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T17:17:08,293 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:17:08,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:17:08,294 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:17:08,295 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): f5a2a3f1964bd8ecfa7908952eaec708/A is initiating minor compaction (all files) 2024-12-03T17:17:08,295 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:17:08,295 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): f5a2a3f1964bd8ecfa7908952eaec708/B is initiating minor compaction (all files) 2024-12-03T17:17:08,295 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5a2a3f1964bd8ecfa7908952eaec708/A in TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:08,295 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5a2a3f1964bd8ecfa7908952eaec708/B in TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:08,295 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/8e0255fa618443c09bc32cbc2c6b801a, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/9a4dbc9e644241fb8ccf622d11240846, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/72a37d808998408bb4125610b33eefd0] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp, totalSize=36.8 K 2024-12-03T17:17:08,295 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/ff82b4ff19034de4b7904b2d1e2b9f52, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/067656658ae342cea905705a39facd18, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/cc73b2aac4244ebaa1d055c4d87207a3] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp, totalSize=36.8 K 2024-12-03T17:17:08,296 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e0255fa618443c09bc32cbc2c6b801a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1733246224980 2024-12-03T17:17:08,296 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff82b4ff19034de4b7904b2d1e2b9f52, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1733246224980 2024-12-03T17:17:08,296 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 067656658ae342cea905705a39facd18, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1733246225416 2024-12-03T17:17:08,296 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a4dbc9e644241fb8ccf622d11240846, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1733246225416 2024-12-03T17:17:08,297 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc73b2aac4244ebaa1d055c4d87207a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1733246227211 2024-12-03T17:17:08,297 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 72a37d808998408bb4125610b33eefd0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1733246227211 2024-12-03T17:17:08,313 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5a2a3f1964bd8ecfa7908952eaec708#A#compaction#234 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:17:08,315 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/c35e47378d74400db4fc8336fd1cc96d is 50, key is test_row_0/A:col10/1733246227211/Put/seqid=0 2024-12-03T17:17:08,320 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5a2a3f1964bd8ecfa7908952eaec708#B#compaction#235 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:17:08,325 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/4644e0be5c634f00ac508893ba7d996d is 50, key is test_row_0/B:col10/1733246227211/Put/seqid=0 2024-12-03T17:17:08,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742101_1277 (size=13221) 2024-12-03T17:17:08,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742100_1276 (size=13221) 2024-12-03T17:17:08,383 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/4644e0be5c634f00ac508893ba7d996d as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/4644e0be5c634f00ac508893ba7d996d 2024-12-03T17:17:08,384 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f5a2a3f1964bd8ecfa7908952eaec708 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-03T17:17:08,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on f5a2a3f1964bd8ecfa7908952eaec708 2024-12-03T17:17:08,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=A 2024-12-03T17:17:08,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:08,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=B 2024-12-03T17:17:08,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:08,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=C 2024-12-03T17:17:08,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:08,394 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5a2a3f1964bd8ecfa7908952eaec708/B of f5a2a3f1964bd8ecfa7908952eaec708 into 4644e0be5c634f00ac508893ba7d996d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:17:08,395 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:17:08,395 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., storeName=f5a2a3f1964bd8ecfa7908952eaec708/B, priority=13, startTime=1733246228293; duration=0sec 2024-12-03T17:17:08,395 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:17:08,395 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5a2a3f1964bd8ecfa7908952eaec708:B 2024-12-03T17:17:08,395 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:17:08,399 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/08b71ebbee5c4c978c0b46d63fe58eff is 50, key is test_row_0/A:col10/1733246228378/Put/seqid=0 2024-12-03T17:17:08,400 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:17:08,400 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): f5a2a3f1964bd8ecfa7908952eaec708/C is initiating minor compaction (all files) 2024-12-03T17:17:08,400 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5a2a3f1964bd8ecfa7908952eaec708/C in TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:08,400 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/574543a2850c4fb0bb1074c9c2fee4eb, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/2988a9efb43f40fab35a3bdbcf488908, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/4c630e530ea1445898703f42db9fb623] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp, totalSize=36.8 K 2024-12-03T17:17:08,401 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 574543a2850c4fb0bb1074c9c2fee4eb, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1733246224980 2024-12-03T17:17:08,403 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 2988a9efb43f40fab35a3bdbcf488908, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1733246225416 2024-12-03T17:17:08,404 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/c35e47378d74400db4fc8336fd1cc96d as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/c35e47378d74400db4fc8336fd1cc96d 2024-12-03T17:17:08,404 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c630e530ea1445898703f42db9fb623, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1733246227211 2024-12-03T17:17:08,416 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5a2a3f1964bd8ecfa7908952eaec708/A of f5a2a3f1964bd8ecfa7908952eaec708 into c35e47378d74400db4fc8336fd1cc96d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:17:08,416 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:17:08,416 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., storeName=f5a2a3f1964bd8ecfa7908952eaec708/A, priority=13, startTime=1733246228293; duration=0sec 2024-12-03T17:17:08,417 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:08,417 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5a2a3f1964bd8ecfa7908952eaec708:A 2024-12-03T17:17:08,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742102_1278 (size=14741) 2024-12-03T17:17:08,435 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5a2a3f1964bd8ecfa7908952eaec708#C#compaction#237 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:17:08,436 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/440d55f36b35417f8db32ebfa414be1d is 50, key is test_row_0/C:col10/1733246227211/Put/seqid=0 2024-12-03T17:17:08,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:08,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246288431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:08,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:08,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:08,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:08,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:08,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246288437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:08,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:08,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246288438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:08,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:08,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246288443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:08,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:08,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:08,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:08,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:08,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:08,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:08,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:08,452 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:08,452 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:08,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742103_1279 (size=13221) 2024-12-03T17:17:08,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:08,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246288543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:08,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:08,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:08,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:08,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:08,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246288554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:08,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:08,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246288555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:08,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:08,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:08,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:08,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:08,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:08,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:08,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:08,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246288754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:08,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:08,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:08,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:08,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:08,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246288767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:08,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:08,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:08,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:08,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:08,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246288775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:08,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:08,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:08,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:08,839 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/08b71ebbee5c4c978c0b46d63fe58eff 2024-12-03T17:17:08,893 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/51ab58928f604488a37c49193b259acd is 50, key is test_row_0/B:col10/1733246228378/Put/seqid=0 2024-12-03T17:17:08,907 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/440d55f36b35417f8db32ebfa414be1d as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/440d55f36b35417f8db32ebfa414be1d 2024-12-03T17:17:08,935 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5a2a3f1964bd8ecfa7908952eaec708/C of f5a2a3f1964bd8ecfa7908952eaec708 into 440d55f36b35417f8db32ebfa414be1d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:17:08,935 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:17:08,935 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., storeName=f5a2a3f1964bd8ecfa7908952eaec708/C, priority=13, startTime=1733246228293; duration=0sec 2024-12-03T17:17:08,935 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:08,935 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5a2a3f1964bd8ecfa7908952eaec708:C 2024-12-03T17:17:08,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742104_1280 (size=12301) 2024-12-03T17:17:08,949 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/51ab58928f604488a37c49193b259acd 2024-12-03T17:17:08,966 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/5a02b3d2d9584a8fad6c258e70bde110 is 50, key is test_row_0/C:col10/1733246228378/Put/seqid=0 2024-12-03T17:17:08,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:08,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246288968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:08,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:08,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:08,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:08,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742105_1281 (size=12301) 2024-12-03T17:17:08,982 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/5a02b3d2d9584a8fad6c258e70bde110 2024-12-03T17:17:08,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/08b71ebbee5c4c978c0b46d63fe58eff as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/08b71ebbee5c4c978c0b46d63fe58eff 2024-12-03T17:17:09,024 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/08b71ebbee5c4c978c0b46d63fe58eff, entries=200, sequenceid=409, filesize=14.4 K 2024-12-03T17:17:09,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/51ab58928f604488a37c49193b259acd as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/51ab58928f604488a37c49193b259acd 2024-12-03T17:17:09,056 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/51ab58928f604488a37c49193b259acd, entries=150, sequenceid=409, filesize=12.0 K 2024-12-03T17:17:09,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/5a02b3d2d9584a8fad6c258e70bde110 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/5a02b3d2d9584a8fad6c258e70bde110 2024-12-03T17:17:09,067 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/5a02b3d2d9584a8fad6c258e70bde110, entries=150, sequenceid=409, filesize=12.0 K 2024-12-03T17:17:09,068 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for f5a2a3f1964bd8ecfa7908952eaec708 in 684ms, sequenceid=409, compaction requested=false 2024-12-03T17:17:09,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:17:09,088 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f5a2a3f1964bd8ecfa7908952eaec708 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-03T17:17:09,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=A 2024-12-03T17:17:09,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:09,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=B 2024-12-03T17:17:09,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:09,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=C 2024-12-03T17:17:09,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:09,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on f5a2a3f1964bd8ecfa7908952eaec708 2024-12-03T17:17:09,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/616b5cd88b554d33bac8246561b2fc7f is 50, key is test_row_0/A:col10/1733246229084/Put/seqid=0 2024-12-03T17:17:09,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-03T17:17:09,119 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:17:09,120 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T17:17:09,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-12-03T17:17:09,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-03T17:17:09,122 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T17:17:09,124 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T17:17:09,124 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T17:17:09,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742106_1282 (size=9857) 2024-12-03T17:17:09,145 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/616b5cd88b554d33bac8246561b2fc7f 2024-12-03T17:17:09,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:09,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246289180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:09,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:09,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246289180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:09,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:09,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:09,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:09,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:09,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246289181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:09,190 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:09,190 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:09,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:09,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/769b3fd09877416d9def8fe121f689ca is 50, key is test_row_0/B:col10/1733246229084/Put/seqid=0 2024-12-03T17:17:09,196 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:09,196 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:09,196 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:09,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-03T17:17:09,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742107_1283 (size=9857) 2024-12-03T17:17:09,243 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/769b3fd09877416d9def8fe121f689ca 2024-12-03T17:17:09,267 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/0d43cee1837448e8945e71efaa5d8584 is 50, key is test_row_0/C:col10/1733246229084/Put/seqid=0 2024-12-03T17:17:09,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-03T17:17:09,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:09,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. as already flushing 2024-12-03T17:17:09,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:09,283 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:09,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:09,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:09,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:09,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246289293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:09,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:09,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:09,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:09,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:09,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246289295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:09,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:09,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:09,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:09,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742108_1284 (size=9857) 2024-12-03T17:17:09,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:09,313 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/0d43cee1837448e8945e71efaa5d8584 2024-12-03T17:17:09,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246289303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:09,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:09,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:09,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:09,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/616b5cd88b554d33bac8246561b2fc7f as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/616b5cd88b554d33bac8246561b2fc7f 2024-12-03T17:17:09,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/616b5cd88b554d33bac8246561b2fc7f, entries=100, sequenceid=429, filesize=9.6 K 2024-12-03T17:17:09,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/769b3fd09877416d9def8fe121f689ca as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/769b3fd09877416d9def8fe121f689ca 2024-12-03T17:17:09,374 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/769b3fd09877416d9def8fe121f689ca, entries=100, sequenceid=429, filesize=9.6 K 2024-12-03T17:17:09,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/0d43cee1837448e8945e71efaa5d8584 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/0d43cee1837448e8945e71efaa5d8584 2024-12-03T17:17:09,385 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/0d43cee1837448e8945e71efaa5d8584, entries=100, sequenceid=429, filesize=9.6 K 2024-12-03T17:17:09,386 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for f5a2a3f1964bd8ecfa7908952eaec708 in 298ms, sequenceid=429, compaction requested=true 2024-12-03T17:17:09,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:17:09,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5a2a3f1964bd8ecfa7908952eaec708:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T17:17:09,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:09,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5a2a3f1964bd8ecfa7908952eaec708:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T17:17:09,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:09,386 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:17:09,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5a2a3f1964bd8ecfa7908952eaec708:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T17:17:09,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:17:09,386 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:17:09,398 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35379 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:17:09,399 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37819 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:17:09,400 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): f5a2a3f1964bd8ecfa7908952eaec708/A is initiating minor compaction (all files) 2024-12-03T17:17:09,400 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5a2a3f1964bd8ecfa7908952eaec708/A in TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:09,400 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/c35e47378d74400db4fc8336fd1cc96d, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/08b71ebbee5c4c978c0b46d63fe58eff, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/616b5cd88b554d33bac8246561b2fc7f] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp, totalSize=36.9 K 2024-12-03T17:17:09,403 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting c35e47378d74400db4fc8336fd1cc96d, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1733246227211 2024-12-03T17:17:09,403 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08b71ebbee5c4c978c0b46d63fe58eff, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1733246227686 2024-12-03T17:17:09,405 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 616b5cd88b554d33bac8246561b2fc7f, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1733246228429 2024-12-03T17:17:09,411 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): f5a2a3f1964bd8ecfa7908952eaec708/B is initiating minor compaction (all files) 2024-12-03T17:17:09,411 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5a2a3f1964bd8ecfa7908952eaec708/B in TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:09,411 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/4644e0be5c634f00ac508893ba7d996d, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/51ab58928f604488a37c49193b259acd, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/769b3fd09877416d9def8fe121f689ca] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp, totalSize=34.5 K 2024-12-03T17:17:09,415 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 4644e0be5c634f00ac508893ba7d996d, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1733246227211 2024-12-03T17:17:09,417 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 51ab58928f604488a37c49193b259acd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1733246227764 2024-12-03T17:17:09,419 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 769b3fd09877416d9def8fe121f689ca, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1733246228429 2024-12-03T17:17:09,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-03T17:17:09,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:09,443 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing f5a2a3f1964bd8ecfa7908952eaec708 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-03T17:17:09,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=A 2024-12-03T17:17:09,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:09,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=B 2024-12-03T17:17:09,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:09,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=C 2024-12-03T17:17:09,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:09,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-03T17:17:09,466 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5a2a3f1964bd8ecfa7908952eaec708#A#compaction#243 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:17:09,466 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/4c92c242fae2495ab130076cb5d2e941 is 50, key is test_row_0/A:col10/1733246229084/Put/seqid=0 2024-12-03T17:17:09,476 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5a2a3f1964bd8ecfa7908952eaec708#B#compaction#244 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:17:09,477 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/9fc2138cb58647179b6aa225777f5753 is 50, key is test_row_0/B:col10/1733246229084/Put/seqid=0 2024-12-03T17:17:09,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/3041075ae32a4a17935496b86aa50080 is 50, key is test_row_0/A:col10/1733246229175/Put/seqid=0 2024-12-03T17:17:09,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on f5a2a3f1964bd8ecfa7908952eaec708 2024-12-03T17:17:09,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. as already flushing 2024-12-03T17:17:09,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742110_1286 (size=13323) 2024-12-03T17:17:09,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742109_1285 (size=13323) 2024-12-03T17:17:09,575 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/9fc2138cb58647179b6aa225777f5753 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/9fc2138cb58647179b6aa225777f5753 2024-12-03T17:17:09,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742111_1287 (size=12301) 2024-12-03T17:17:09,586 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/3041075ae32a4a17935496b86aa50080 2024-12-03T17:17:09,593 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5a2a3f1964bd8ecfa7908952eaec708/B of f5a2a3f1964bd8ecfa7908952eaec708 into 9fc2138cb58647179b6aa225777f5753(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:17:09,593 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:17:09,593 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., storeName=f5a2a3f1964bd8ecfa7908952eaec708/B, priority=13, startTime=1733246229386; duration=0sec 2024-12-03T17:17:09,593 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:17:09,593 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5a2a3f1964bd8ecfa7908952eaec708:B 2024-12-03T17:17:09,593 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:17:09,599 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35379 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:17:09,599 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): f5a2a3f1964bd8ecfa7908952eaec708/C is initiating minor compaction (all files) 2024-12-03T17:17:09,599 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5a2a3f1964bd8ecfa7908952eaec708/C in TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:09,599 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/440d55f36b35417f8db32ebfa414be1d, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/5a02b3d2d9584a8fad6c258e70bde110, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/0d43cee1837448e8945e71efaa5d8584] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp, totalSize=34.5 K 2024-12-03T17:17:09,603 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 440d55f36b35417f8db32ebfa414be1d, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1733246227211 2024-12-03T17:17:09,604 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a02b3d2d9584a8fad6c258e70bde110, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1733246227764 2024-12-03T17:17:09,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/f1a738b4dee544ab8ef6898c2105c979 is 50, key is test_row_0/B:col10/1733246229175/Put/seqid=0 2024-12-03T17:17:09,607 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d43cee1837448e8945e71efaa5d8584, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1733246228429 2024-12-03T17:17:09,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:09,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246289599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:09,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:09,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:09,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:09,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:09,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246289605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:09,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:09,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246289609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:09,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:09,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:09,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:09,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:09,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:09,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:09,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742112_1288 (size=12301) 2024-12-03T17:17:09,677 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5a2a3f1964bd8ecfa7908952eaec708#C#compaction#247 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:17:09,678 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/6eed9034c3634d0d807a70b6044c01b6 is 50, key is test_row_0/C:col10/1733246229084/Put/seqid=0 2024-12-03T17:17:09,679 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/f1a738b4dee544ab8ef6898c2105c979 2024-12-03T17:17:09,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-03T17:17:09,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:09,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246289743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:09,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:09,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246289751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:09,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:09,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246289751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:09,758 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:09,758 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:09,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:09,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:09,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:09,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:09,760 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:09,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/0ec26ef1d1b5461287eb5145de52f14e is 50, key is test_row_0/C:col10/1733246229175/Put/seqid=0 2024-12-03T17:17:09,760 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:09,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:09,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742113_1289 (size=13323) 2024-12-03T17:17:09,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742114_1290 (size=12301) 2024-12-03T17:17:09,806 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/6eed9034c3634d0d807a70b6044c01b6 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/6eed9034c3634d0d807a70b6044c01b6 2024-12-03T17:17:09,807 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/0ec26ef1d1b5461287eb5145de52f14e 2024-12-03T17:17:09,815 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5a2a3f1964bd8ecfa7908952eaec708/C of f5a2a3f1964bd8ecfa7908952eaec708 into 6eed9034c3634d0d807a70b6044c01b6(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:17:09,815 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:17:09,815 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., storeName=f5a2a3f1964bd8ecfa7908952eaec708/C, priority=13, startTime=1733246229386; duration=0sec 2024-12-03T17:17:09,815 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:09,815 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5a2a3f1964bd8ecfa7908952eaec708:C 2024-12-03T17:17:09,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/3041075ae32a4a17935496b86aa50080 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/3041075ae32a4a17935496b86aa50080 2024-12-03T17:17:09,830 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/3041075ae32a4a17935496b86aa50080, entries=150, sequenceid=444, filesize=12.0 K 2024-12-03T17:17:09,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/f1a738b4dee544ab8ef6898c2105c979 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/f1a738b4dee544ab8ef6898c2105c979 2024-12-03T17:17:09,837 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/f1a738b4dee544ab8ef6898c2105c979, entries=150, sequenceid=444, filesize=12.0 K 2024-12-03T17:17:09,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/0ec26ef1d1b5461287eb5145de52f14e as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/0ec26ef1d1b5461287eb5145de52f14e 2024-12-03T17:17:09,843 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/0ec26ef1d1b5461287eb5145de52f14e, entries=150, sequenceid=444, filesize=12.0 K 2024-12-03T17:17:09,844 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for f5a2a3f1964bd8ecfa7908952eaec708 in 401ms, sequenceid=444, compaction requested=false 2024-12-03T17:17:09,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:17:09,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:09,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-12-03T17:17:09,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-12-03T17:17:09,855 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-12-03T17:17:09,855 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 726 msec 2024-12-03T17:17:09,862 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 740 msec 2024-12-03T17:17:09,973 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/4c92c242fae2495ab130076cb5d2e941 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/4c92c242fae2495ab130076cb5d2e941 2024-12-03T17:17:09,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on f5a2a3f1964bd8ecfa7908952eaec708 2024-12-03T17:17:09,975 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f5a2a3f1964bd8ecfa7908952eaec708 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-03T17:17:09,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=A 2024-12-03T17:17:09,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:09,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=B 2024-12-03T17:17:09,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:09,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=C 2024-12-03T17:17:09,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:09,989 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5a2a3f1964bd8ecfa7908952eaec708/A of f5a2a3f1964bd8ecfa7908952eaec708 into 4c92c242fae2495ab130076cb5d2e941(size=13.0 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:17:09,989 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:17:09,989 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., storeName=f5a2a3f1964bd8ecfa7908952eaec708/A, priority=13, startTime=1733246229386; duration=0sec 2024-12-03T17:17:09,989 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:09,990 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5a2a3f1964bd8ecfa7908952eaec708:A 2024-12-03T17:17:10,000 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/1619d631709c44e682ae53449733a274 is 50, key is test_row_0/A:col10/1733246229973/Put/seqid=0 2024-12-03T17:17:10,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742115_1291 (size=14741) 2024-12-03T17:17:10,050 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/1619d631709c44e682ae53449733a274 2024-12-03T17:17:10,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:10,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246290033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:10,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:10,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:10,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:10,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:10,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246290053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:10,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:10,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246290053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:10,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:10,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:10,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:10,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:10,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246290060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:10,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:10,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:10,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:10,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:10,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:10,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:10,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/5429bddabbcf455cacc6e33c2356b136 is 50, key is test_row_0/B:col10/1733246229973/Put/seqid=0 2024-12-03T17:17:10,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742116_1292 (size=12301) 2024-12-03T17:17:10,143 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/5429bddabbcf455cacc6e33c2356b136 2024-12-03T17:17:10,160 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/7f3f315cc3df4b05a306e0317dfa4f0a is 50, key is test_row_0/C:col10/1733246229973/Put/seqid=0 2024-12-03T17:17:10,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:10,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246290164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:10,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:10,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:10,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:10,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:10,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246290174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:10,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:10,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:10,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:10,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:10,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246290174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:10,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:10,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:10,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246290174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:10,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:10,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:10,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:10,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:10,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:10,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742117_1293 (size=12301) 2024-12-03T17:17:10,206 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/7f3f315cc3df4b05a306e0317dfa4f0a 2024-12-03T17:17:10,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/1619d631709c44e682ae53449733a274 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/1619d631709c44e682ae53449733a274 2024-12-03T17:17:10,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/1619d631709c44e682ae53449733a274, entries=200, sequenceid=465, filesize=14.4 K 2024-12-03T17:17:10,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/5429bddabbcf455cacc6e33c2356b136 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/5429bddabbcf455cacc6e33c2356b136 2024-12-03T17:17:10,239 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/5429bddabbcf455cacc6e33c2356b136, entries=150, sequenceid=465, filesize=12.0 K 2024-12-03T17:17:10,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/7f3f315cc3df4b05a306e0317dfa4f0a as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/7f3f315cc3df4b05a306e0317dfa4f0a 2024-12-03T17:17:10,251 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/7f3f315cc3df4b05a306e0317dfa4f0a, entries=150, sequenceid=465, filesize=12.0 K 2024-12-03T17:17:10,255 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for f5a2a3f1964bd8ecfa7908952eaec708 in 280ms, sequenceid=465, compaction requested=true 2024-12-03T17:17:10,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:17:10,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5a2a3f1964bd8ecfa7908952eaec708:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T17:17:10,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:10,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5a2a3f1964bd8ecfa7908952eaec708:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T17:17:10,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:17:10,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5a2a3f1964bd8ecfa7908952eaec708:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T17:17:10,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T17:17:10,256 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:17:10,258 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:17:10,259 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40365 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:17:10,259 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): f5a2a3f1964bd8ecfa7908952eaec708/A is initiating minor compaction (all files) 2024-12-03T17:17:10,259 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5a2a3f1964bd8ecfa7908952eaec708/A in TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:10,259 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/4c92c242fae2495ab130076cb5d2e941, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/3041075ae32a4a17935496b86aa50080, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/1619d631709c44e682ae53449733a274] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp, totalSize=39.4 K 2024-12-03T17:17:10,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-03T17:17:10,264 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:17:10,265 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T17:17:10,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-12-03T17:17:10,267 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c92c242fae2495ab130076cb5d2e941, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1733246228384 2024-12-03T17:17:10,267 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T17:17:10,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-03T17:17:10,267 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:17:10,267 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 3041075ae32a4a17935496b86aa50080, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1733246229156 2024-12-03T17:17:10,267 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): f5a2a3f1964bd8ecfa7908952eaec708/B is initiating minor compaction (all files) 2024-12-03T17:17:10,267 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5a2a3f1964bd8ecfa7908952eaec708/B in TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:10,268 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/9fc2138cb58647179b6aa225777f5753, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/f1a738b4dee544ab8ef6898c2105c979, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/5429bddabbcf455cacc6e33c2356b136] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp, totalSize=37.0 K 2024-12-03T17:17:10,268 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T17:17:10,268 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 1619d631709c44e682ae53449733a274, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1733246229596 2024-12-03T17:17:10,268 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T17:17:10,268 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fc2138cb58647179b6aa225777f5753, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1733246228384 2024-12-03T17:17:10,269 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1a738b4dee544ab8ef6898c2105c979, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1733246229156 2024-12-03T17:17:10,269 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5429bddabbcf455cacc6e33c2356b136, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1733246229598 2024-12-03T17:17:10,288 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5a2a3f1964bd8ecfa7908952eaec708#A#compaction#252 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:17:10,289 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/6a1fd6fa215847e3b0e27b355b2bf784 is 50, key is test_row_0/A:col10/1733246229973/Put/seqid=0 2024-12-03T17:17:10,292 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5a2a3f1964bd8ecfa7908952eaec708#B#compaction#253 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:17:10,292 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/0bf16d695efe4b549486b6b36e735cf5 is 50, key is test_row_0/B:col10/1733246229973/Put/seqid=0 2024-12-03T17:17:10,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742118_1294 (size=13425) 2024-12-03T17:17:10,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742119_1295 (size=13425) 2024-12-03T17:17:10,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-03T17:17:10,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on f5a2a3f1964bd8ecfa7908952eaec708 2024-12-03T17:17:10,381 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f5a2a3f1964bd8ecfa7908952eaec708 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-03T17:17:10,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=A 2024-12-03T17:17:10,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:10,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=B 2024-12-03T17:17:10,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:10,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=C 2024-12-03T17:17:10,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:10,383 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/0bf16d695efe4b549486b6b36e735cf5 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/0bf16d695efe4b549486b6b36e735cf5 2024-12-03T17:17:10,397 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5a2a3f1964bd8ecfa7908952eaec708/B of f5a2a3f1964bd8ecfa7908952eaec708 into 0bf16d695efe4b549486b6b36e735cf5(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:17:10,397 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:17:10,397 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., storeName=f5a2a3f1964bd8ecfa7908952eaec708/B, priority=13, startTime=1733246230255; duration=0sec 2024-12-03T17:17:10,398 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:17:10,398 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5a2a3f1964bd8ecfa7908952eaec708:B 2024-12-03T17:17:10,398 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:17:10,401 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:17:10,401 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/3c901659100d4b9882206627bea128ec is 50, key is test_row_0/A:col10/1733246230378/Put/seqid=0 2024-12-03T17:17:10,401 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): f5a2a3f1964bd8ecfa7908952eaec708/C is initiating minor compaction (all files) 2024-12-03T17:17:10,401 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5a2a3f1964bd8ecfa7908952eaec708/C in TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:10,401 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/6eed9034c3634d0d807a70b6044c01b6, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/0ec26ef1d1b5461287eb5145de52f14e, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/7f3f315cc3df4b05a306e0317dfa4f0a] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp, totalSize=37.0 K 2024-12-03T17:17:10,403 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6eed9034c3634d0d807a70b6044c01b6, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1733246228384 2024-12-03T17:17:10,403 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ec26ef1d1b5461287eb5145de52f14e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1733246229156 2024-12-03T17:17:10,404 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f3f315cc3df4b05a306e0317dfa4f0a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1733246229598 2024-12-03T17:17:10,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-03T17:17:10,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:10,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. as already flushing 2024-12-03T17:17:10,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:10,424 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:10,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:10,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:10,436 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5a2a3f1964bd8ecfa7908952eaec708#C#compaction#255 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:17:10,437 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/2f716ed6007546b0b9add189c0ade8f4 is 50, key is test_row_0/C:col10/1733246229973/Put/seqid=0 2024-12-03T17:17:10,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:10,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246290429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:10,443 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:10,443 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:10,443 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:10,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742120_1296 (size=14741) 2024-12-03T17:17:10,445 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/3c901659100d4b9882206627bea128ec 2024-12-03T17:17:10,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:10,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246290441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:10,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:10,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246290441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:10,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:10,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:10,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:10,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:10,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:10,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:10,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:10,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246290446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:10,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:10,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:10,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:10,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742121_1297 (size=13425) 2024-12-03T17:17:10,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/23a3a8bf4c214e86a6cd92103528a707 is 50, key is test_row_0/B:col10/1733246230378/Put/seqid=0 2024-12-03T17:17:10,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742122_1298 (size=12301) 2024-12-03T17:17:10,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:10,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246290555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:10,560 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:10,560 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:10,560 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:10,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:10,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246290563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:10,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:10,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:10,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:10,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:10,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246290563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:10,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:10,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246290564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:10,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:10,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:10,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:10,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:10,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:10,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:10,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-03T17:17:10,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-03T17:17:10,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:10,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. as already flushing 2024-12-03T17:17:10,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:10,585 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:10,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:10,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:10,731 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/6a1fd6fa215847e3b0e27b355b2bf784 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/6a1fd6fa215847e3b0e27b355b2bf784 2024-12-03T17:17:10,736 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5a2a3f1964bd8ecfa7908952eaec708/A of f5a2a3f1964bd8ecfa7908952eaec708 into 6a1fd6fa215847e3b0e27b355b2bf784(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:17:10,736 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:17:10,736 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., storeName=f5a2a3f1964bd8ecfa7908952eaec708/A, priority=13, startTime=1733246230255; duration=0sec 2024-12-03T17:17:10,736 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:10,736 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5a2a3f1964bd8ecfa7908952eaec708:A 2024-12-03T17:17:10,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-03T17:17:10,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:10,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. as already flushing 2024-12-03T17:17:10,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:10,738 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:10,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:10,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:10,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:10,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246290774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:10,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:10,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:10,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:10,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:10,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246290774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:10,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:10,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246290774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:10,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:10,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:10,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:10,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:10,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:10,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:10,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:10,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246290774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:10,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:10,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:10,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:10,893 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/2f716ed6007546b0b9add189c0ade8f4 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/2f716ed6007546b0b9add189c0ade8f4 2024-12-03T17:17:10,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-03T17:17:10,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-03T17:17:10,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:10,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. as already flushing 2024-12-03T17:17:10,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:10,897 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:10,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:10,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:10,913 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5a2a3f1964bd8ecfa7908952eaec708/C of f5a2a3f1964bd8ecfa7908952eaec708 into 2f716ed6007546b0b9add189c0ade8f4(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:17:10,913 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:17:10,913 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., storeName=f5a2a3f1964bd8ecfa7908952eaec708/C, priority=13, startTime=1733246230255; duration=0sec 2024-12-03T17:17:10,914 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:10,914 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5a2a3f1964bd8ecfa7908952eaec708:C 2024-12-03T17:17:10,922 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/23a3a8bf4c214e86a6cd92103528a707 2024-12-03T17:17:10,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/acf233d0149048bc9b6e27f87b2ffe64 is 50, key is test_row_0/C:col10/1733246230378/Put/seqid=0 2024-12-03T17:17:10,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742123_1299 (size=12301) 2024-12-03T17:17:10,969 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/acf233d0149048bc9b6e27f87b2ffe64 2024-12-03T17:17:10,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/3c901659100d4b9882206627bea128ec as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/3c901659100d4b9882206627bea128ec 2024-12-03T17:17:10,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/3c901659100d4b9882206627bea128ec, entries=200, sequenceid=481, filesize=14.4 K 2024-12-03T17:17:10,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/23a3a8bf4c214e86a6cd92103528a707 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/23a3a8bf4c214e86a6cd92103528a707 2024-12-03T17:17:11,004 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/23a3a8bf4c214e86a6cd92103528a707, entries=150, sequenceid=481, filesize=12.0 K 2024-12-03T17:17:11,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/acf233d0149048bc9b6e27f87b2ffe64 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/acf233d0149048bc9b6e27f87b2ffe64 2024-12-03T17:17:11,012 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/acf233d0149048bc9b6e27f87b2ffe64, entries=150, sequenceid=481, filesize=12.0 K 2024-12-03T17:17:11,014 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for f5a2a3f1964bd8ecfa7908952eaec708 in 633ms, sequenceid=481, compaction requested=false 2024-12-03T17:17:11,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:17:11,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-03T17:17:11,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:11,061 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing f5a2a3f1964bd8ecfa7908952eaec708 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-03T17:17:11,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=A 2024-12-03T17:17:11,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:11,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=B 2024-12-03T17:17:11,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:11,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=C 2024-12-03T17:17:11,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:11,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/f8a0696ac8854446b128362110139433 is 50, key is test_row_0/A:col10/1733246230414/Put/seqid=0 2024-12-03T17:17:11,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on f5a2a3f1964bd8ecfa7908952eaec708 2024-12-03T17:17:11,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. as already flushing 2024-12-03T17:17:11,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742124_1300 (size=12301) 2024-12-03T17:17:11,122 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/f8a0696ac8854446b128362110139433 2024-12-03T17:17:11,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:11,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246291122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:11,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:11,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:11,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:11,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:11,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246291135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:11,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/7d64c15526714698bf0ee60f84e74a96 is 50, key is test_row_0/B:col10/1733246230414/Put/seqid=0 2024-12-03T17:17:11,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:11,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:11,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:11,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:11,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246291145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:11,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:11,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:11,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:11,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:11,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246291156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:11,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:11,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:11,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:11,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742125_1301 (size=12301) 2024-12-03T17:17:11,199 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/7d64c15526714698bf0ee60f84e74a96 2024-12-03T17:17:11,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/0d82c02158b2444a9062b261c7f2218d is 50, key is test_row_0/C:col10/1733246230414/Put/seqid=0 2024-12-03T17:17:11,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:11,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246291243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:11,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:11,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:11,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:11,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:11,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246291264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:11,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:11,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246291264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:11,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:11,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:11,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:11,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:11,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:11,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:11,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742126_1302 (size=12301) 2024-12-03T17:17:11,280 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/0d82c02158b2444a9062b261c7f2218d 2024-12-03T17:17:11,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:11,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246291273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:11,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:11,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:11,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:11,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/f8a0696ac8854446b128362110139433 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/f8a0696ac8854446b128362110139433 2024-12-03T17:17:11,298 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/f8a0696ac8854446b128362110139433, entries=150, sequenceid=502, filesize=12.0 K 2024-12-03T17:17:11,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/7d64c15526714698bf0ee60f84e74a96 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/7d64c15526714698bf0ee60f84e74a96 2024-12-03T17:17:11,326 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/7d64c15526714698bf0ee60f84e74a96, entries=150, sequenceid=502, filesize=12.0 K 2024-12-03T17:17:11,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/0d82c02158b2444a9062b261c7f2218d as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/0d82c02158b2444a9062b261c7f2218d 2024-12-03T17:17:11,333 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/0d82c02158b2444a9062b261c7f2218d, entries=150, sequenceid=502, filesize=12.0 K 2024-12-03T17:17:11,351 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for f5a2a3f1964bd8ecfa7908952eaec708 in 290ms, sequenceid=502, compaction requested=true 2024-12-03T17:17:11,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:17:11,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:11,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-12-03T17:17:11,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-12-03T17:17:11,359 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-12-03T17:17:11,359 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0870 sec 2024-12-03T17:17:11,361 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.0950 sec 2024-12-03T17:17:11,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-03T17:17:11,423 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:17:11,424 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T17:17:11,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-12-03T17:17:11,426 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T17:17:11,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-03T17:17:11,428 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T17:17:11,428 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T17:17:11,470 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f5a2a3f1964bd8ecfa7908952eaec708 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-03T17:17:11,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=A 2024-12-03T17:17:11,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:11,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=B 2024-12-03T17:17:11,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:11,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=C 2024-12-03T17:17:11,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:11,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on f5a2a3f1964bd8ecfa7908952eaec708 2024-12-03T17:17:11,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/983bb98b45d64ed4be0b74d6d5cd965d is 50, key is test_row_0/A:col10/1733246231096/Put/seqid=0 2024-12-03T17:17:11,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742127_1303 (size=12301) 2024-12-03T17:17:11,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:11,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246291510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:11,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:11,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246291511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:11,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:11,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:11,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:11,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:11,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246291513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:11,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:11,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:11,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:11,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:11,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246291516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:11,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:11,523 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:11,523 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:11,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:11,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:11,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:11,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-03T17:17:11,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-03T17:17:11,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:11,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. as already flushing 2024-12-03T17:17:11,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:11,596 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:11,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:11,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:11,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:11,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246291624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:11,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:11,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:11,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:11,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:11,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246291624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:11,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:11,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:11,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:11,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:11,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246291633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:11,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:11,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246291633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:11,642 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:11,642 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:11,642 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:11,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:11,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:11,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:11,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-03T17:17:11,751 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-03T17:17:11,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:11,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. as already flushing 2024-12-03T17:17:11,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:11,751 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:11,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:11,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:11,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:11,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246291834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:11,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:11,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:11,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:11,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:11,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246291844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:11,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:11,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246291844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:11,851 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:11,851 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:11,851 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:11,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:11,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:11,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:11,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:11,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246291853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:11,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:11,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:11,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:11,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-03T17:17:11,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:11,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. as already flushing 2024-12-03T17:17:11,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:11,907 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:11,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:11,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:11,915 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=518 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/983bb98b45d64ed4be0b74d6d5cd965d 2024-12-03T17:17:11,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/662e963de66b4330b94dc4e03c69b302 is 50, key is test_row_0/B:col10/1733246231096/Put/seqid=0 2024-12-03T17:17:11,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742128_1304 (size=12301) 2024-12-03T17:17:12,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-03T17:17:12,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-03T17:17:12,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:12,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. as already flushing 2024-12-03T17:17:12,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:12,062 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:12,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:12,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:12,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:12,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246292146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:12,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:12,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:12,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:12,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:12,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246292165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:12,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:12,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:12,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:12,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:12,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246292165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:12,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:12,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:12,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:12,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:12,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246292166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:12,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:12,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:12,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:12,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-03T17:17:12,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:12,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. as already flushing 2024-12-03T17:17:12,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:12,224 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:12,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:12,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:12,373 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=518 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/662e963de66b4330b94dc4e03c69b302 2024-12-03T17:17:12,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-03T17:17:12,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:12,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. as already flushing 2024-12-03T17:17:12,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:12,378 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:12,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:12,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:12,400 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/224558d47f3445399825040c07af8f28 is 50, key is test_row_0/C:col10/1733246231096/Put/seqid=0 2024-12-03T17:17:12,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742129_1305 (size=12301) 2024-12-03T17:17:12,449 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=518 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/224558d47f3445399825040c07af8f28 2024-12-03T17:17:12,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/983bb98b45d64ed4be0b74d6d5cd965d as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/983bb98b45d64ed4be0b74d6d5cd965d 2024-12-03T17:17:12,493 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/983bb98b45d64ed4be0b74d6d5cd965d, entries=150, sequenceid=518, filesize=12.0 K 2024-12-03T17:17:12,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/662e963de66b4330b94dc4e03c69b302 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/662e963de66b4330b94dc4e03c69b302 2024-12-03T17:17:12,501 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/662e963de66b4330b94dc4e03c69b302, entries=150, sequenceid=518, filesize=12.0 K 2024-12-03T17:17:12,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/224558d47f3445399825040c07af8f28 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/224558d47f3445399825040c07af8f28 2024-12-03T17:17:12,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/224558d47f3445399825040c07af8f28, entries=150, sequenceid=518, filesize=12.0 K 2024-12-03T17:17:12,512 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for f5a2a3f1964bd8ecfa7908952eaec708 in 1042ms, sequenceid=518, compaction requested=true 2024-12-03T17:17:12,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:17:12,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5a2a3f1964bd8ecfa7908952eaec708:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T17:17:12,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:12,512 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T17:17:12,512 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T17:17:12,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5a2a3f1964bd8ecfa7908952eaec708:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T17:17:12,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:12,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5a2a3f1964bd8ecfa7908952eaec708:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T17:17:12,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:17:12,519 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52768 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T17:17:12,519 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50328 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T17:17:12,519 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): f5a2a3f1964bd8ecfa7908952eaec708/A is initiating minor compaction (all files) 2024-12-03T17:17:12,519 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): f5a2a3f1964bd8ecfa7908952eaec708/B is initiating minor compaction (all files) 2024-12-03T17:17:12,519 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5a2a3f1964bd8ecfa7908952eaec708/A in TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:12,519 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5a2a3f1964bd8ecfa7908952eaec708/B in TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:12,519 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/6a1fd6fa215847e3b0e27b355b2bf784, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/3c901659100d4b9882206627bea128ec, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/f8a0696ac8854446b128362110139433, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/983bb98b45d64ed4be0b74d6d5cd965d] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp, totalSize=51.5 K 2024-12-03T17:17:12,519 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/0bf16d695efe4b549486b6b36e735cf5, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/23a3a8bf4c214e86a6cd92103528a707, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/7d64c15526714698bf0ee60f84e74a96, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/662e963de66b4330b94dc4e03c69b302] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp, totalSize=49.1 K 2024-12-03T17:17:12,520 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bf16d695efe4b549486b6b36e735cf5, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1733246229598 2024-12-03T17:17:12,520 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a1fd6fa215847e3b0e27b355b2bf784, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1733246229598 2024-12-03T17:17:12,520 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c901659100d4b9882206627bea128ec, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1733246230017 2024-12-03T17:17:12,521 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8a0696ac8854446b128362110139433, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1733246230414 2024-12-03T17:17:12,521 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 23a3a8bf4c214e86a6cd92103528a707, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1733246230041 2024-12-03T17:17:12,522 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 983bb98b45d64ed4be0b74d6d5cd965d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=518, earliestPutTs=1733246231096 2024-12-03T17:17:12,522 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d64c15526714698bf0ee60f84e74a96, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1733246230414 2024-12-03T17:17:12,532 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 662e963de66b4330b94dc4e03c69b302, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=518, earliestPutTs=1733246231096 2024-12-03T17:17:12,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-03T17:17:12,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:12,535 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing f5a2a3f1964bd8ecfa7908952eaec708 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-03T17:17:12,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=A 2024-12-03T17:17:12,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:12,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=B 2024-12-03T17:17:12,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:12,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=C 2024-12-03T17:17:12,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:12,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/70dd82ddbdc14a65b9d8fc94a2de68ee is 50, key is test_row_0/A:col10/1733246231499/Put/seqid=0 2024-12-03T17:17:12,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-03T17:17:12,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742130_1306 (size=12301) 2024-12-03T17:17:12,607 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/70dd82ddbdc14a65b9d8fc94a2de68ee 2024-12-03T17:17:12,611 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5a2a3f1964bd8ecfa7908952eaec708#B#compaction#265 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:17:12,612 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/39ae623851394b35a51b1e7de072beee is 50, key is test_row_0/B:col10/1733246231096/Put/seqid=0 2024-12-03T17:17:12,615 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5a2a3f1964bd8ecfa7908952eaec708#A#compaction#266 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:17:12,616 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/7e49bddfde954634b923ef7a07fa3983 is 50, key is test_row_0/A:col10/1733246231096/Put/seqid=0 2024-12-03T17:17:12,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/b1f0b9056c7346388a3510a8c9b19c3a is 50, key is test_row_0/B:col10/1733246231499/Put/seqid=0 2024-12-03T17:17:12,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742131_1307 (size=13561) 2024-12-03T17:17:12,663 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/39ae623851394b35a51b1e7de072beee as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/39ae623851394b35a51b1e7de072beee 2024-12-03T17:17:12,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on f5a2a3f1964bd8ecfa7908952eaec708 2024-12-03T17:17:12,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. as already flushing 2024-12-03T17:17:12,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742132_1308 (size=13561) 2024-12-03T17:17:12,677 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f5a2a3f1964bd8ecfa7908952eaec708/B of f5a2a3f1964bd8ecfa7908952eaec708 into 39ae623851394b35a51b1e7de072beee(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:17:12,677 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:17:12,677 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., storeName=f5a2a3f1964bd8ecfa7908952eaec708/B, priority=12, startTime=1733246232512; duration=0sec 2024-12-03T17:17:12,677 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:17:12,677 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5a2a3f1964bd8ecfa7908952eaec708:B 2024-12-03T17:17:12,677 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T17:17:12,687 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50328 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T17:17:12,687 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): f5a2a3f1964bd8ecfa7908952eaec708/C is initiating minor compaction (all files) 2024-12-03T17:17:12,688 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5a2a3f1964bd8ecfa7908952eaec708/C in TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:12,688 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/2f716ed6007546b0b9add189c0ade8f4, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/acf233d0149048bc9b6e27f87b2ffe64, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/0d82c02158b2444a9062b261c7f2218d, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/224558d47f3445399825040c07af8f28] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp, totalSize=49.1 K 2024-12-03T17:17:12,690 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f716ed6007546b0b9add189c0ade8f4, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1733246229598 2024-12-03T17:17:12,692 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting acf233d0149048bc9b6e27f87b2ffe64, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1733246230041 2024-12-03T17:17:12,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742133_1309 (size=12301) 2024-12-03T17:17:12,693 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d82c02158b2444a9062b261c7f2218d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1733246230414 2024-12-03T17:17:12,696 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 224558d47f3445399825040c07af8f28, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=518, earliestPutTs=1733246231096 2024-12-03T17:17:12,698 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/7e49bddfde954634b923ef7a07fa3983 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/7e49bddfde954634b923ef7a07fa3983 2024-12-03T17:17:12,699 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/b1f0b9056c7346388a3510a8c9b19c3a 2024-12-03T17:17:12,709 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f5a2a3f1964bd8ecfa7908952eaec708/A of f5a2a3f1964bd8ecfa7908952eaec708 into 7e49bddfde954634b923ef7a07fa3983(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:17:12,709 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:17:12,709 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., storeName=f5a2a3f1964bd8ecfa7908952eaec708/A, priority=12, startTime=1733246232512; duration=0sec 2024-12-03T17:17:12,709 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:12,709 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5a2a3f1964bd8ecfa7908952eaec708:A 2024-12-03T17:17:12,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/b7a8585b0a5444b48a87dbf720f25c72 is 50, key is test_row_0/C:col10/1733246231499/Put/seqid=0 2024-12-03T17:17:12,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:12,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42088 deadline: 1733246292709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:12,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:12,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:12,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:12,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:12,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:12,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42102 deadline: 1733246292716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:12,729 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5a2a3f1964bd8ecfa7908952eaec708#C#compaction#269 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:17:12,730 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/5ba5cecdd0ac48bb82da7902a4464b49 is 50, key is test_row_0/C:col10/1733246231096/Put/seqid=0 2024-12-03T17:17:12,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42066 deadline: 1733246292716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:12,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:12,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:12,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:12,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:12,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:12,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:12,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:12,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42110 deadline: 1733246292717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:12,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:12,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:12,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:12,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742135_1311 (size=13561) 2024-12-03T17:17:12,775 INFO [Thread-951 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-951. 2024-12-03T17:17:12,775 DEBUG [Thread-951 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:17:12,775 DEBUG [Thread-951 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:12,775 DEBUG [Thread-951 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:12,777 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:17:12,778 INFO [Thread-945 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-945. 2024-12-03T17:17:12,778 DEBUG [Thread-945 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:17:12,778 DEBUG [Thread-945 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:12,778 DEBUG [Thread-945 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:12,778 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:17:12,780 INFO [Thread-949 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-949. 2024-12-03T17:17:12,780 DEBUG [Thread-949 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:17:12,780 DEBUG [Thread-949 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:12,780 DEBUG [Thread-949 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:12,780 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:17:12,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742134_1310 (size=12301) 2024-12-03T17:17:12,782 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/b7a8585b0a5444b48a87dbf720f25c72 2024-12-03T17:17:12,782 INFO [Thread-953 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-953. 2024-12-03T17:17:12,782 DEBUG [Thread-953 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:17:12,782 DEBUG [Thread-953 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:12,783 DEBUG [Thread-953 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:12,783 INFO [Thread-947 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-947. 2024-12-03T17:17:12,783 DEBUG [Thread-947 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:17:12,783 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:17:12,783 DEBUG [Thread-947 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:12,783 DEBUG [Thread-947 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:12,783 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:17:12,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/70dd82ddbdc14a65b9d8fc94a2de68ee as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/70dd82ddbdc14a65b9d8fc94a2de68ee 2024-12-03T17:17:12,795 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/70dd82ddbdc14a65b9d8fc94a2de68ee, entries=150, sequenceid=534, filesize=12.0 K 2024-12-03T17:17:12,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/b1f0b9056c7346388a3510a8c9b19c3a as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/b1f0b9056c7346388a3510a8c9b19c3a 2024-12-03T17:17:12,801 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/b1f0b9056c7346388a3510a8c9b19c3a, entries=150, sequenceid=534, filesize=12.0 K 2024-12-03T17:17:12,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/b7a8585b0a5444b48a87dbf720f25c72 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/b7a8585b0a5444b48a87dbf720f25c72 2024-12-03T17:17:12,808 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/b7a8585b0a5444b48a87dbf720f25c72, entries=150, sequenceid=534, filesize=12.0 K 2024-12-03T17:17:12,809 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for f5a2a3f1964bd8ecfa7908952eaec708 in 274ms, sequenceid=534, compaction requested=false 2024-12-03T17:17:12,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:17:12,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:12,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-12-03T17:17:12,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-12-03T17:17:12,812 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-12-03T17:17:12,812 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3830 sec 2024-12-03T17:17:12,814 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.3890 sec 2024-12-03T17:17:12,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on f5a2a3f1964bd8ecfa7908952eaec708 2024-12-03T17:17:12,826 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f5a2a3f1964bd8ecfa7908952eaec708 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-03T17:17:12,827 INFO [Thread-934 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-934. 2024-12-03T17:17:12,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=A 2024-12-03T17:17:12,827 DEBUG [Thread-934 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:17:12,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:12,827 DEBUG [Thread-934 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:12,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=B 2024-12-03T17:17:12,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:12,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=C 2024-12-03T17:17:12,827 DEBUG [Thread-934 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:12,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:12,827 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:17:12,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/b7f357fab2e2449d863cedf616542ce5 is 50, key is test_row_0/A:col10/1733246232824/Put/seqid=0 2024-12-03T17:17:12,835 INFO [Thread-942 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-942. 2024-12-03T17:17:12,835 DEBUG [Thread-942 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:17:12,835 DEBUG [Thread-942 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:12,835 DEBUG [Thread-942 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:12,835 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:17:12,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742136_1312 (size=12301) 2024-12-03T17:17:12,845 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=553 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/b7f357fab2e2449d863cedf616542ce5 2024-12-03T17:17:12,846 INFO [Thread-940 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-940. 2024-12-03T17:17:12,846 DEBUG [Thread-940 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:17:12,846 DEBUG [Thread-940 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:12,846 DEBUG [Thread-940 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:12,847 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:17:12,851 INFO [Thread-938 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-938. 2024-12-03T17:17:12,851 DEBUG [Thread-938 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:17:12,852 DEBUG [Thread-938 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:12,852 DEBUG [Thread-938 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:12,852 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:17:12,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/8ed6af6e1bee48489766ab0a17778d42 is 50, key is test_row_0/B:col10/1733246232824/Put/seqid=0 2024-12-03T17:17:12,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742137_1313 (size=12301) 2024-12-03T17:17:13,178 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/5ba5cecdd0ac48bb82da7902a4464b49 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/5ba5cecdd0ac48bb82da7902a4464b49 2024-12-03T17:17:13,216 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f5a2a3f1964bd8ecfa7908952eaec708/C of f5a2a3f1964bd8ecfa7908952eaec708 into 5ba5cecdd0ac48bb82da7902a4464b49(size=13.2 K), total size for store is 25.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:17:13,216 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:17:13,216 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., storeName=f5a2a3f1964bd8ecfa7908952eaec708/C, priority=12, startTime=1733246232513; duration=0sec 2024-12-03T17:17:13,217 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:13,217 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5a2a3f1964bd8ecfa7908952eaec708:C 2024-12-03T17:17:13,262 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=553 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/8ed6af6e1bee48489766ab0a17778d42 2024-12-03T17:17:13,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/20beeaa50e5646fe89770f7d281b3ce8 is 50, key is test_row_0/C:col10/1733246232824/Put/seqid=0 2024-12-03T17:17:13,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742138_1314 (size=12301) 2024-12-03T17:17:13,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-03T17:17:13,574 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:17:13,677 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=553 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/20beeaa50e5646fe89770f7d281b3ce8 2024-12-03T17:17:13,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/b7f357fab2e2449d863cedf616542ce5 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/b7f357fab2e2449d863cedf616542ce5 2024-12-03T17:17:13,685 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/b7f357fab2e2449d863cedf616542ce5, entries=150, sequenceid=553, filesize=12.0 K 2024-12-03T17:17:13,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/8ed6af6e1bee48489766ab0a17778d42 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/8ed6af6e1bee48489766ab0a17778d42 2024-12-03T17:17:13,689 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/8ed6af6e1bee48489766ab0a17778d42, entries=150, sequenceid=553, filesize=12.0 K 2024-12-03T17:17:13,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/20beeaa50e5646fe89770f7d281b3ce8 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/20beeaa50e5646fe89770f7d281b3ce8 2024-12-03T17:17:13,693 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/20beeaa50e5646fe89770f7d281b3ce8, entries=150, sequenceid=553, filesize=12.0 K 2024-12-03T17:17:13,694 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=20.13 KB/20610 for f5a2a3f1964bd8ecfa7908952eaec708 in 868ms, sequenceid=553, compaction requested=true 2024-12-03T17:17:13,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:17:13,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5a2a3f1964bd8ecfa7908952eaec708:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T17:17:13,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:13,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5a2a3f1964bd8ecfa7908952eaec708:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T17:17:13,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:13,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f5a2a3f1964bd8ecfa7908952eaec708:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T17:17:13,694 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:17:13,694 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:17:13,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:17:13,695 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38163 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:17:13,695 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): f5a2a3f1964bd8ecfa7908952eaec708/B is initiating minor compaction (all files) 2024-12-03T17:17:13,695 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38163 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:17:13,695 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5a2a3f1964bd8ecfa7908952eaec708/B in TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:13,695 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): f5a2a3f1964bd8ecfa7908952eaec708/A is initiating minor compaction (all files) 2024-12-03T17:17:13,695 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5a2a3f1964bd8ecfa7908952eaec708/A in TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:13,695 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/39ae623851394b35a51b1e7de072beee, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/b1f0b9056c7346388a3510a8c9b19c3a, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/8ed6af6e1bee48489766ab0a17778d42] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp, totalSize=37.3 K 2024-12-03T17:17:13,695 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/7e49bddfde954634b923ef7a07fa3983, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/70dd82ddbdc14a65b9d8fc94a2de68ee, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/b7f357fab2e2449d863cedf616542ce5] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp, totalSize=37.3 K 2024-12-03T17:17:13,696 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 39ae623851394b35a51b1e7de072beee, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=518, earliestPutTs=1733246231096 2024-12-03T17:17:13,696 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e49bddfde954634b923ef7a07fa3983, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=518, earliestPutTs=1733246231096 2024-12-03T17:17:13,696 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting b1f0b9056c7346388a3510a8c9b19c3a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1733246231499 2024-12-03T17:17:13,696 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70dd82ddbdc14a65b9d8fc94a2de68ee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1733246231499 2024-12-03T17:17:13,696 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7f357fab2e2449d863cedf616542ce5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=553, earliestPutTs=1733246232705 2024-12-03T17:17:13,696 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ed6af6e1bee48489766ab0a17778d42, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=553, earliestPutTs=1733246232705 2024-12-03T17:17:13,704 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5a2a3f1964bd8ecfa7908952eaec708#A#compaction#273 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:17:13,704 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/76fd2c8d536e414f96f62d0480a7edd2 is 50, key is test_row_0/A:col10/1733246232824/Put/seqid=0 2024-12-03T17:17:13,704 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5a2a3f1964bd8ecfa7908952eaec708#B#compaction#274 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:17:13,705 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/4c0b18012da64953989673279c290ff0 is 50, key is test_row_0/B:col10/1733246232824/Put/seqid=0 2024-12-03T17:17:13,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742139_1315 (size=13663) 2024-12-03T17:17:13,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742140_1316 (size=13663) 2024-12-03T17:17:13,713 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/4c0b18012da64953989673279c290ff0 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/4c0b18012da64953989673279c290ff0 2024-12-03T17:17:13,717 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5a2a3f1964bd8ecfa7908952eaec708/B of f5a2a3f1964bd8ecfa7908952eaec708 into 4c0b18012da64953989673279c290ff0(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:17:13,717 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:17:13,717 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., storeName=f5a2a3f1964bd8ecfa7908952eaec708/B, priority=13, startTime=1733246233694; duration=0sec 2024-12-03T17:17:13,717 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:17:13,717 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5a2a3f1964bd8ecfa7908952eaec708:B 2024-12-03T17:17:13,717 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:17:13,718 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38163 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:17:13,718 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): f5a2a3f1964bd8ecfa7908952eaec708/C is initiating minor compaction (all files) 2024-12-03T17:17:13,718 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f5a2a3f1964bd8ecfa7908952eaec708/C in TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:13,718 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/5ba5cecdd0ac48bb82da7902a4464b49, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/b7a8585b0a5444b48a87dbf720f25c72, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/20beeaa50e5646fe89770f7d281b3ce8] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp, totalSize=37.3 K 2024-12-03T17:17:13,719 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ba5cecdd0ac48bb82da7902a4464b49, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=518, earliestPutTs=1733246231096 2024-12-03T17:17:13,719 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting b7a8585b0a5444b48a87dbf720f25c72, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1733246231499 2024-12-03T17:17:13,719 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 20beeaa50e5646fe89770f7d281b3ce8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=553, earliestPutTs=1733246232705 2024-12-03T17:17:13,726 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f5a2a3f1964bd8ecfa7908952eaec708#C#compaction#275 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:17:13,727 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/a62e3675ff0d4beb8ffd5b2b81ccf6b5 is 50, key is test_row_0/C:col10/1733246232824/Put/seqid=0 2024-12-03T17:17:13,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742141_1317 (size=13663) 2024-12-03T17:17:14,113 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/76fd2c8d536e414f96f62d0480a7edd2 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/76fd2c8d536e414f96f62d0480a7edd2 2024-12-03T17:17:14,118 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5a2a3f1964bd8ecfa7908952eaec708/A of f5a2a3f1964bd8ecfa7908952eaec708 into 76fd2c8d536e414f96f62d0480a7edd2(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:17:14,118 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:17:14,118 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., storeName=f5a2a3f1964bd8ecfa7908952eaec708/A, priority=13, startTime=1733246233694; duration=0sec 2024-12-03T17:17:14,119 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:14,119 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5a2a3f1964bd8ecfa7908952eaec708:A 2024-12-03T17:17:14,136 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/a62e3675ff0d4beb8ffd5b2b81ccf6b5 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/a62e3675ff0d4beb8ffd5b2b81ccf6b5 2024-12-03T17:17:14,141 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f5a2a3f1964bd8ecfa7908952eaec708/C of f5a2a3f1964bd8ecfa7908952eaec708 into a62e3675ff0d4beb8ffd5b2b81ccf6b5(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:17:14,141 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f5a2a3f1964bd8ecfa7908952eaec708: 2024-12-03T17:17:14,141 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., storeName=f5a2a3f1964bd8ecfa7908952eaec708/C, priority=13, startTime=1733246233694; duration=0sec 2024-12-03T17:17:14,141 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:14,141 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f5a2a3f1964bd8ecfa7908952eaec708:C 2024-12-03T17:17:17,648 INFO [Thread-936 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-936. 2024-12-03T17:17:17,648 DEBUG [Thread-936 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:17:17,648 DEBUG [Thread-936 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:17,648 DEBUG [Thread-936 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:17,649 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:17:17,649 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-03T17:17:17,649 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 110 2024-12-03T17:17:17,649 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 32 2024-12-03T17:17:17,649 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 101 2024-12-03T17:17:17,649 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 94 2024-12-03T17:17:17,649 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 100 2024-12-03T17:17:17,649 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-03T17:17:17,649 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1686 2024-12-03T17:17:17,649 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1634 2024-12-03T17:17:17,649 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1665 2024-12-03T17:17:17,649 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1704 2024-12-03T17:17:17,649 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1659 2024-12-03T17:17:17,649 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-03T17:17:17,649 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T17:17:17,649 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T17:17:17,649 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:17,650 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:17,650 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:17:17,654 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-03T17:17:17,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-03T17:17:17,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-03T17:17:17,676 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733246237676"}]},"ts":"1733246237676"} 2024-12-03T17:17:17,682 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-03T17:17:17,682 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-03T17:17:17,686 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T17:17:17,688 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f5a2a3f1964bd8ecfa7908952eaec708, UNASSIGN}] 2024-12-03T17:17:17,689 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f5a2a3f1964bd8ecfa7908952eaec708, UNASSIGN 2024-12-03T17:17:17,690 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=f5a2a3f1964bd8ecfa7908952eaec708, regionState=CLOSING, regionLocation=27e0f852a164,41167,1733246151593 2024-12-03T17:17:17,693 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f5a2a3f1964bd8ecfa7908952eaec708, UNASSIGN because future has completed 2024-12-03T17:17:17,697 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T17:17:17,697 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; CloseRegionProcedure f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593}] 2024-12-03T17:17:17,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-03T17:17:17,855 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] handler.UnassignRegionHandler(122): Close f5a2a3f1964bd8ecfa7908952eaec708 2024-12-03T17:17:17,855 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T17:17:17,855 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1722): Closing f5a2a3f1964bd8ecfa7908952eaec708, disabling compactions & flushes 2024-12-03T17:17:17,855 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:17,855 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:17,855 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. after waiting 0 ms 2024-12-03T17:17:17,855 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:17,855 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(2902): Flushing f5a2a3f1964bd8ecfa7908952eaec708 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-03T17:17:17,856 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=A 2024-12-03T17:17:17,856 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:17,856 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=B 2024-12-03T17:17:17,856 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:17,856 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f5a2a3f1964bd8ecfa7908952eaec708, store=C 2024-12-03T17:17:17,856 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:17,888 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/4213fc1652e647ebaf3ebd2f78b504d3 is 50, key is test_row_0/A:col10/1733246232845/Put/seqid=0 2024-12-03T17:17:17,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742142_1318 (size=12301) 2024-12-03T17:17:17,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-03T17:17:18,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-03T17:17:18,327 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=564 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/4213fc1652e647ebaf3ebd2f78b504d3 2024-12-03T17:17:18,351 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/aa1e2335eeb44ffabcac342bb3e6f5b2 is 50, key is test_row_0/B:col10/1733246232845/Put/seqid=0 2024-12-03T17:17:18,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742143_1319 (size=12301) 2024-12-03T17:17:18,801 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=564 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/aa1e2335eeb44ffabcac342bb3e6f5b2 2024-12-03T17:17:18,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-03T17:17:18,827 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/49a730a52d354b6386dd60ef5b015b64 is 50, key is test_row_0/C:col10/1733246232845/Put/seqid=0 2024-12-03T17:17:18,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742144_1320 (size=12301) 2024-12-03T17:17:18,842 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=564 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/49a730a52d354b6386dd60ef5b015b64 2024-12-03T17:17:18,859 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/A/4213fc1652e647ebaf3ebd2f78b504d3 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/4213fc1652e647ebaf3ebd2f78b504d3 2024-12-03T17:17:18,864 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/4213fc1652e647ebaf3ebd2f78b504d3, entries=150, sequenceid=564, filesize=12.0 K 2024-12-03T17:17:18,866 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/B/aa1e2335eeb44ffabcac342bb3e6f5b2 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/aa1e2335eeb44ffabcac342bb3e6f5b2 2024-12-03T17:17:18,872 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/aa1e2335eeb44ffabcac342bb3e6f5b2, entries=150, sequenceid=564, filesize=12.0 K 2024-12-03T17:17:18,873 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/.tmp/C/49a730a52d354b6386dd60ef5b015b64 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/49a730a52d354b6386dd60ef5b015b64 2024-12-03T17:17:18,878 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/49a730a52d354b6386dd60ef5b015b64, entries=150, sequenceid=564, filesize=12.0 K 2024-12-03T17:17:18,879 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for f5a2a3f1964bd8ecfa7908952eaec708 in 1024ms, sequenceid=564, compaction requested=false 2024-12-03T17:17:18,880 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/f2f2f8b7324943c08d4248641513361b, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/c6b1f679a8ae4ba39a4e9eacf058f1b8, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/0de1fb8f9ad747e8a6a6aaf432a4c37e, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/582fb99ef9f84933a179be6da7b5192d, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/cfb11057901e43659c5de495a3c289de, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/5b62d3fbb9a24130865294f06fe6bd43, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/ad1c140c58e74febb55311230e64e66a, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/465ed8ccd6c5415b9f1897267c2d7cff, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/0b77a685fb7a423d8903655a55e3a13c, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/2fb94978b2f54b278d2753fcae9ca9de, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/81fec17dff884914b2b8a19f7f46caf3, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/a8eeee1d804443ca81b0646500b95d84, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/b9ebb1d73d014b028170a371f764a51a, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/61686ee92e964c1c83e01b0ef3022ad5, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/ee22dd4b461a419b897c6badbe1748e9, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/e78e8de45a984e10932d91409c5cfa6f, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/8003769579f0499f8d2f2dd2db288285, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/ad2f8b21e9d94f288d248b0501b06f91, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/47c3f81fa98e41cabd0957726e82a6c1, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/2d19e751a69145b8a8509ef3d5d8f0f8, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/c0ff985e679f4f58bc9b278ae4d7d3e0, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/49827d2dff7f4f9f897b0430760c0d54, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/6d521ab4e22f4962ae132139210d567d, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/ff82b4ff19034de4b7904b2d1e2b9f52, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/76113bb8a795464a9ffa90a809c96bed, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/067656658ae342cea905705a39facd18, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/c35e47378d74400db4fc8336fd1cc96d, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/cc73b2aac4244ebaa1d055c4d87207a3, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/08b71ebbee5c4c978c0b46d63fe58eff, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/4c92c242fae2495ab130076cb5d2e941, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/616b5cd88b554d33bac8246561b2fc7f, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/3041075ae32a4a17935496b86aa50080, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/1619d631709c44e682ae53449733a274, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/6a1fd6fa215847e3b0e27b355b2bf784, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/3c901659100d4b9882206627bea128ec, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/f8a0696ac8854446b128362110139433, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/7e49bddfde954634b923ef7a07fa3983, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/983bb98b45d64ed4be0b74d6d5cd965d, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/70dd82ddbdc14a65b9d8fc94a2de68ee, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/b7f357fab2e2449d863cedf616542ce5] to archive 2024-12-03T17:17:18,882 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T17:17:18,885 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/f2f2f8b7324943c08d4248641513361b to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/f2f2f8b7324943c08d4248641513361b 2024-12-03T17:17:18,889 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/c6b1f679a8ae4ba39a4e9eacf058f1b8 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/c6b1f679a8ae4ba39a4e9eacf058f1b8 2024-12-03T17:17:18,890 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/0de1fb8f9ad747e8a6a6aaf432a4c37e to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/0de1fb8f9ad747e8a6a6aaf432a4c37e 2024-12-03T17:17:18,899 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/582fb99ef9f84933a179be6da7b5192d to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/582fb99ef9f84933a179be6da7b5192d 2024-12-03T17:17:18,906 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/cfb11057901e43659c5de495a3c289de to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/cfb11057901e43659c5de495a3c289de 2024-12-03T17:17:18,928 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/5b62d3fbb9a24130865294f06fe6bd43 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/5b62d3fbb9a24130865294f06fe6bd43 2024-12-03T17:17:18,933 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/ad1c140c58e74febb55311230e64e66a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/ad1c140c58e74febb55311230e64e66a 2024-12-03T17:17:18,935 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/465ed8ccd6c5415b9f1897267c2d7cff to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/465ed8ccd6c5415b9f1897267c2d7cff 2024-12-03T17:17:18,954 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/0b77a685fb7a423d8903655a55e3a13c to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/0b77a685fb7a423d8903655a55e3a13c 2024-12-03T17:17:18,956 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/2fb94978b2f54b278d2753fcae9ca9de to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/2fb94978b2f54b278d2753fcae9ca9de 2024-12-03T17:17:18,958 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/81fec17dff884914b2b8a19f7f46caf3 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/81fec17dff884914b2b8a19f7f46caf3 2024-12-03T17:17:18,961 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/a8eeee1d804443ca81b0646500b95d84 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/a8eeee1d804443ca81b0646500b95d84 2024-12-03T17:17:18,977 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/b9ebb1d73d014b028170a371f764a51a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/b9ebb1d73d014b028170a371f764a51a 2024-12-03T17:17:18,988 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/61686ee92e964c1c83e01b0ef3022ad5 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/61686ee92e964c1c83e01b0ef3022ad5 2024-12-03T17:17:18,999 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/ee22dd4b461a419b897c6badbe1748e9 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/ee22dd4b461a419b897c6badbe1748e9 2024-12-03T17:17:19,011 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/e78e8de45a984e10932d91409c5cfa6f to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/e78e8de45a984e10932d91409c5cfa6f 2024-12-03T17:17:19,018 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/8003769579f0499f8d2f2dd2db288285 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/8003769579f0499f8d2f2dd2db288285 2024-12-03T17:17:19,022 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/ad2f8b21e9d94f288d248b0501b06f91 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/ad2f8b21e9d94f288d248b0501b06f91 2024-12-03T17:17:19,032 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/47c3f81fa98e41cabd0957726e82a6c1 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/47c3f81fa98e41cabd0957726e82a6c1 2024-12-03T17:17:19,089 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/2d19e751a69145b8a8509ef3d5d8f0f8 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/2d19e751a69145b8a8509ef3d5d8f0f8 2024-12-03T17:17:19,091 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/c0ff985e679f4f58bc9b278ae4d7d3e0 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/c0ff985e679f4f58bc9b278ae4d7d3e0 2024-12-03T17:17:19,093 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/49827d2dff7f4f9f897b0430760c0d54 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/49827d2dff7f4f9f897b0430760c0d54 2024-12-03T17:17:19,096 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/6d521ab4e22f4962ae132139210d567d to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/6d521ab4e22f4962ae132139210d567d 2024-12-03T17:17:19,108 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/ff82b4ff19034de4b7904b2d1e2b9f52 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/ff82b4ff19034de4b7904b2d1e2b9f52 2024-12-03T17:17:19,110 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/76113bb8a795464a9ffa90a809c96bed to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/76113bb8a795464a9ffa90a809c96bed 2024-12-03T17:17:19,112 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/067656658ae342cea905705a39facd18 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/067656658ae342cea905705a39facd18 2024-12-03T17:17:19,115 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/c35e47378d74400db4fc8336fd1cc96d to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/c35e47378d74400db4fc8336fd1cc96d 2024-12-03T17:17:19,123 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/cc73b2aac4244ebaa1d055c4d87207a3 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/cc73b2aac4244ebaa1d055c4d87207a3 2024-12-03T17:17:19,129 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/08b71ebbee5c4c978c0b46d63fe58eff to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/08b71ebbee5c4c978c0b46d63fe58eff 2024-12-03T17:17:19,139 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/4c92c242fae2495ab130076cb5d2e941 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/4c92c242fae2495ab130076cb5d2e941 2024-12-03T17:17:19,152 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/616b5cd88b554d33bac8246561b2fc7f to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/616b5cd88b554d33bac8246561b2fc7f 2024-12-03T17:17:19,154 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/3041075ae32a4a17935496b86aa50080 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/3041075ae32a4a17935496b86aa50080 2024-12-03T17:17:19,155 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/1619d631709c44e682ae53449733a274 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/1619d631709c44e682ae53449733a274 2024-12-03T17:17:19,157 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/6a1fd6fa215847e3b0e27b355b2bf784 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/6a1fd6fa215847e3b0e27b355b2bf784 2024-12-03T17:17:19,165 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/3c901659100d4b9882206627bea128ec to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/3c901659100d4b9882206627bea128ec 2024-12-03T17:17:19,171 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/f8a0696ac8854446b128362110139433 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/f8a0696ac8854446b128362110139433 2024-12-03T17:17:19,174 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/7e49bddfde954634b923ef7a07fa3983 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/7e49bddfde954634b923ef7a07fa3983 2024-12-03T17:17:19,175 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/983bb98b45d64ed4be0b74d6d5cd965d to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/983bb98b45d64ed4be0b74d6d5cd965d 2024-12-03T17:17:19,177 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/70dd82ddbdc14a65b9d8fc94a2de68ee to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/70dd82ddbdc14a65b9d8fc94a2de68ee 2024-12-03T17:17:19,179 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/b7f357fab2e2449d863cedf616542ce5 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/b7f357fab2e2449d863cedf616542ce5 2024-12-03T17:17:19,181 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/44016882368743f18d498a980d752fc9, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/f352954592ce42de8bd97ad80267a801, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/99e3f78412914425bac8cf76ebf8ee48, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/e997d37380d04c96b9c978121c5dd6ee, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/5b0cea4aedeb483995ce7116e6694d62, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/3ba29def7138414d873495722cb9adf5, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/12b7b6fd1ea3404a85e80c7fb8f34bb5, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/27c369e20f62429f8b14f1e7259a7fd7, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/f079c3008ae84b72875e236b04e02148, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/a4e7622e443b41e980007297754eb32b, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/a5ef0e9a526d4faf8fe1387106609417, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/885e5819c0814da7bd72b68d4be9f458, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/78dc38a468de48db9f7acdaf560638ed, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/8f417c16d2504653ae22989da192189a, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/1cf45fad67184f99acd1fc26bb6210e9, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/5f28ad23023b4f0db8b606be5d4c5009, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/f376bfaec8d940f6ab9c445fd30313e3, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/4d2d00110cbd412ab00e377cd77379b4, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/d8e60dde9a534a08bb5c188a685c7a9f, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/01c35a2eed7a42fdbbe97d4767cfdda2, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/f2c9e62fb7674d2ab72a4881cf4892dc, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/7b866d16484f4b78aba71a7028746b8e, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/4182e6e291234e749dce865f1302a127, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/8e0255fa618443c09bc32cbc2c6b801a, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/831cbafc416e4c47acdc4e9bb6ac4fc2, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/9a4dbc9e644241fb8ccf622d11240846, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/4644e0be5c634f00ac508893ba7d996d, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/72a37d808998408bb4125610b33eefd0, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/51ab58928f604488a37c49193b259acd, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/9fc2138cb58647179b6aa225777f5753, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/769b3fd09877416d9def8fe121f689ca, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/f1a738b4dee544ab8ef6898c2105c979, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/0bf16d695efe4b549486b6b36e735cf5, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/5429bddabbcf455cacc6e33c2356b136, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/23a3a8bf4c214e86a6cd92103528a707, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/7d64c15526714698bf0ee60f84e74a96, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/39ae623851394b35a51b1e7de072beee, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/662e963de66b4330b94dc4e03c69b302, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/b1f0b9056c7346388a3510a8c9b19c3a, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/8ed6af6e1bee48489766ab0a17778d42] to archive 2024-12-03T17:17:19,184 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T17:17:19,198 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/44016882368743f18d498a980d752fc9 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/44016882368743f18d498a980d752fc9 2024-12-03T17:17:19,210 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/f352954592ce42de8bd97ad80267a801 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/f352954592ce42de8bd97ad80267a801 2024-12-03T17:17:19,223 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/99e3f78412914425bac8cf76ebf8ee48 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/99e3f78412914425bac8cf76ebf8ee48 2024-12-03T17:17:19,228 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/e997d37380d04c96b9c978121c5dd6ee to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/e997d37380d04c96b9c978121c5dd6ee 2024-12-03T17:17:19,230 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/5b0cea4aedeb483995ce7116e6694d62 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/5b0cea4aedeb483995ce7116e6694d62 2024-12-03T17:17:19,231 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/3ba29def7138414d873495722cb9adf5 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/3ba29def7138414d873495722cb9adf5 2024-12-03T17:17:19,233 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/12b7b6fd1ea3404a85e80c7fb8f34bb5 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/12b7b6fd1ea3404a85e80c7fb8f34bb5 2024-12-03T17:17:19,234 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/27c369e20f62429f8b14f1e7259a7fd7 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/27c369e20f62429f8b14f1e7259a7fd7 2024-12-03T17:17:19,236 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/f079c3008ae84b72875e236b04e02148 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/f079c3008ae84b72875e236b04e02148 2024-12-03T17:17:19,237 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/a4e7622e443b41e980007297754eb32b to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/a4e7622e443b41e980007297754eb32b 2024-12-03T17:17:19,245 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/a5ef0e9a526d4faf8fe1387106609417 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/a5ef0e9a526d4faf8fe1387106609417 2024-12-03T17:17:19,253 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/885e5819c0814da7bd72b68d4be9f458 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/885e5819c0814da7bd72b68d4be9f458 2024-12-03T17:17:19,256 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/78dc38a468de48db9f7acdaf560638ed to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/78dc38a468de48db9f7acdaf560638ed 2024-12-03T17:17:19,257 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/8f417c16d2504653ae22989da192189a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/8f417c16d2504653ae22989da192189a 2024-12-03T17:17:19,260 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/1cf45fad67184f99acd1fc26bb6210e9 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/1cf45fad67184f99acd1fc26bb6210e9 2024-12-03T17:17:19,270 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/5f28ad23023b4f0db8b606be5d4c5009 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/5f28ad23023b4f0db8b606be5d4c5009 2024-12-03T17:17:19,284 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/f376bfaec8d940f6ab9c445fd30313e3 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/f376bfaec8d940f6ab9c445fd30313e3 2024-12-03T17:17:19,292 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/4d2d00110cbd412ab00e377cd77379b4 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/4d2d00110cbd412ab00e377cd77379b4 2024-12-03T17:17:19,301 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/d8e60dde9a534a08bb5c188a685c7a9f to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/d8e60dde9a534a08bb5c188a685c7a9f 2024-12-03T17:17:19,314 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/01c35a2eed7a42fdbbe97d4767cfdda2 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/01c35a2eed7a42fdbbe97d4767cfdda2 2024-12-03T17:17:19,316 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/f2c9e62fb7674d2ab72a4881cf4892dc to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/f2c9e62fb7674d2ab72a4881cf4892dc 2024-12-03T17:17:19,317 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/7b866d16484f4b78aba71a7028746b8e to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/7b866d16484f4b78aba71a7028746b8e 2024-12-03T17:17:19,319 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/4182e6e291234e749dce865f1302a127 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/4182e6e291234e749dce865f1302a127 2024-12-03T17:17:19,321 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/8e0255fa618443c09bc32cbc2c6b801a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/8e0255fa618443c09bc32cbc2c6b801a 2024-12-03T17:17:19,322 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/831cbafc416e4c47acdc4e9bb6ac4fc2 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/831cbafc416e4c47acdc4e9bb6ac4fc2 2024-12-03T17:17:19,326 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/9a4dbc9e644241fb8ccf622d11240846 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/9a4dbc9e644241fb8ccf622d11240846 2024-12-03T17:17:19,328 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/4644e0be5c634f00ac508893ba7d996d to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/4644e0be5c634f00ac508893ba7d996d 2024-12-03T17:17:19,330 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/72a37d808998408bb4125610b33eefd0 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/72a37d808998408bb4125610b33eefd0 2024-12-03T17:17:19,334 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/51ab58928f604488a37c49193b259acd to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/51ab58928f604488a37c49193b259acd 2024-12-03T17:17:19,341 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/9fc2138cb58647179b6aa225777f5753 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/9fc2138cb58647179b6aa225777f5753 2024-12-03T17:17:19,350 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/769b3fd09877416d9def8fe121f689ca to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/769b3fd09877416d9def8fe121f689ca 2024-12-03T17:17:19,364 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/f1a738b4dee544ab8ef6898c2105c979 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/f1a738b4dee544ab8ef6898c2105c979 2024-12-03T17:17:19,383 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/0bf16d695efe4b549486b6b36e735cf5 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/0bf16d695efe4b549486b6b36e735cf5 2024-12-03T17:17:19,390 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/5429bddabbcf455cacc6e33c2356b136 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/5429bddabbcf455cacc6e33c2356b136 2024-12-03T17:17:19,392 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/23a3a8bf4c214e86a6cd92103528a707 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/23a3a8bf4c214e86a6cd92103528a707 2024-12-03T17:17:19,393 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/7d64c15526714698bf0ee60f84e74a96 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/7d64c15526714698bf0ee60f84e74a96 2024-12-03T17:17:19,394 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/39ae623851394b35a51b1e7de072beee to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/39ae623851394b35a51b1e7de072beee 2024-12-03T17:17:19,395 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/662e963de66b4330b94dc4e03c69b302 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/662e963de66b4330b94dc4e03c69b302 2024-12-03T17:17:19,396 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/b1f0b9056c7346388a3510a8c9b19c3a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/b1f0b9056c7346388a3510a8c9b19c3a 2024-12-03T17:17:19,397 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/8ed6af6e1bee48489766ab0a17778d42 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/8ed6af6e1bee48489766ab0a17778d42 2024-12-03T17:17:19,399 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T17:17:19,400 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/735f576162194064986a40028fa8098a, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/adabd2f6ca7e4a8d95bde30eaa48e59b, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/55efcb7a565d49f38f2e216350c472ac, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/9a41d48170ba4c219358c6f5a22e3063, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/6e66da2558bc401f8cabac5582cf7083, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/ea747c218a8843348d4dbcb157b352c0, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/6977b13fc6b34ff8960037134802ac29, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/3baf1b2321314a13adf325431c00fd6e, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/0bba7d0abee84966b20619340a2a3c93, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/766711724f5b4b58a2293e249250778b, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/bbfa6e1bf9d64a409ee3a291a54be0c3, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/3fd05673394c4cb899ad99c84fe07bc0, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/525d943a7415472fabfe4d4ef36eaf02, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/426e6e16a74a47e6b888670ac1cfce05, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/19a6dfe130924dbb922359df246161fa, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/5e14e0b166ed433a8df2b3aa9ce46136, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/330e54952bd94ed58b074aa10926d2fa, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/a1f8ca48266749899c6980382cc9c0eb, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/1f330c21ce5445f4a02ecdf5e2ffcc2c, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/257537db81bf4c6bb7837758f4c57a78, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/7eba619ed24f461ab0623f0b4d97bd22, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/dc361fc2cdaf44228b64044faf4a9171, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/fb2d90e170524ff09fd0f9d1663765d4, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/574543a2850c4fb0bb1074c9c2fee4eb, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/4e9e96a5639644d3b45f203b96dccb11, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/2988a9efb43f40fab35a3bdbcf488908, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/440d55f36b35417f8db32ebfa414be1d, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/4c630e530ea1445898703f42db9fb623, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/5a02b3d2d9584a8fad6c258e70bde110, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/6eed9034c3634d0d807a70b6044c01b6, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/0d43cee1837448e8945e71efaa5d8584, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/0ec26ef1d1b5461287eb5145de52f14e, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/2f716ed6007546b0b9add189c0ade8f4, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/7f3f315cc3df4b05a306e0317dfa4f0a, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/acf233d0149048bc9b6e27f87b2ffe64, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/0d82c02158b2444a9062b261c7f2218d, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/5ba5cecdd0ac48bb82da7902a4464b49, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/224558d47f3445399825040c07af8f28, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/b7a8585b0a5444b48a87dbf720f25c72, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/20beeaa50e5646fe89770f7d281b3ce8] to archive 2024-12-03T17:17:19,401 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T17:17:19,402 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/735f576162194064986a40028fa8098a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/735f576162194064986a40028fa8098a 2024-12-03T17:17:19,404 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/adabd2f6ca7e4a8d95bde30eaa48e59b to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/adabd2f6ca7e4a8d95bde30eaa48e59b 2024-12-03T17:17:19,406 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/55efcb7a565d49f38f2e216350c472ac to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/55efcb7a565d49f38f2e216350c472ac 2024-12-03T17:17:19,407 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/9a41d48170ba4c219358c6f5a22e3063 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/9a41d48170ba4c219358c6f5a22e3063 2024-12-03T17:17:19,408 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/6e66da2558bc401f8cabac5582cf7083 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/6e66da2558bc401f8cabac5582cf7083 2024-12-03T17:17:19,409 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/ea747c218a8843348d4dbcb157b352c0 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/ea747c218a8843348d4dbcb157b352c0 2024-12-03T17:17:19,413 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/6977b13fc6b34ff8960037134802ac29 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/6977b13fc6b34ff8960037134802ac29 2024-12-03T17:17:19,415 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/3baf1b2321314a13adf325431c00fd6e to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/3baf1b2321314a13adf325431c00fd6e 2024-12-03T17:17:19,416 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/0bba7d0abee84966b20619340a2a3c93 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/0bba7d0abee84966b20619340a2a3c93 2024-12-03T17:17:19,417 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/766711724f5b4b58a2293e249250778b to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/766711724f5b4b58a2293e249250778b 2024-12-03T17:17:19,418 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/bbfa6e1bf9d64a409ee3a291a54be0c3 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/bbfa6e1bf9d64a409ee3a291a54be0c3 2024-12-03T17:17:19,419 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/3fd05673394c4cb899ad99c84fe07bc0 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/3fd05673394c4cb899ad99c84fe07bc0 2024-12-03T17:17:19,421 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/525d943a7415472fabfe4d4ef36eaf02 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/525d943a7415472fabfe4d4ef36eaf02 2024-12-03T17:17:19,422 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/426e6e16a74a47e6b888670ac1cfce05 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/426e6e16a74a47e6b888670ac1cfce05 2024-12-03T17:17:19,423 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/19a6dfe130924dbb922359df246161fa to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/19a6dfe130924dbb922359df246161fa 2024-12-03T17:17:19,424 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/5e14e0b166ed433a8df2b3aa9ce46136 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/5e14e0b166ed433a8df2b3aa9ce46136 2024-12-03T17:17:19,426 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/330e54952bd94ed58b074aa10926d2fa to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/330e54952bd94ed58b074aa10926d2fa 2024-12-03T17:17:19,427 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/a1f8ca48266749899c6980382cc9c0eb to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/a1f8ca48266749899c6980382cc9c0eb 2024-12-03T17:17:19,428 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/1f330c21ce5445f4a02ecdf5e2ffcc2c to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/1f330c21ce5445f4a02ecdf5e2ffcc2c 2024-12-03T17:17:19,432 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/257537db81bf4c6bb7837758f4c57a78 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/257537db81bf4c6bb7837758f4c57a78 2024-12-03T17:17:19,434 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/7eba619ed24f461ab0623f0b4d97bd22 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/7eba619ed24f461ab0623f0b4d97bd22 2024-12-03T17:17:19,435 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/dc361fc2cdaf44228b64044faf4a9171 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/dc361fc2cdaf44228b64044faf4a9171 2024-12-03T17:17:19,436 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/fb2d90e170524ff09fd0f9d1663765d4 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/fb2d90e170524ff09fd0f9d1663765d4 2024-12-03T17:17:19,438 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/574543a2850c4fb0bb1074c9c2fee4eb to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/574543a2850c4fb0bb1074c9c2fee4eb 2024-12-03T17:17:19,439 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/4e9e96a5639644d3b45f203b96dccb11 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/4e9e96a5639644d3b45f203b96dccb11 2024-12-03T17:17:19,441 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/2988a9efb43f40fab35a3bdbcf488908 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/2988a9efb43f40fab35a3bdbcf488908 2024-12-03T17:17:19,447 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/440d55f36b35417f8db32ebfa414be1d to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/440d55f36b35417f8db32ebfa414be1d 2024-12-03T17:17:19,449 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/4c630e530ea1445898703f42db9fb623 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/4c630e530ea1445898703f42db9fb623 2024-12-03T17:17:19,457 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/5a02b3d2d9584a8fad6c258e70bde110 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/5a02b3d2d9584a8fad6c258e70bde110 2024-12-03T17:17:19,459 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/6eed9034c3634d0d807a70b6044c01b6 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/6eed9034c3634d0d807a70b6044c01b6 2024-12-03T17:17:19,464 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/0d43cee1837448e8945e71efaa5d8584 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/0d43cee1837448e8945e71efaa5d8584 2024-12-03T17:17:19,467 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/0ec26ef1d1b5461287eb5145de52f14e to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/0ec26ef1d1b5461287eb5145de52f14e 2024-12-03T17:17:19,468 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/2f716ed6007546b0b9add189c0ade8f4 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/2f716ed6007546b0b9add189c0ade8f4 2024-12-03T17:17:19,471 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/7f3f315cc3df4b05a306e0317dfa4f0a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/7f3f315cc3df4b05a306e0317dfa4f0a 2024-12-03T17:17:19,472 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/acf233d0149048bc9b6e27f87b2ffe64 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/acf233d0149048bc9b6e27f87b2ffe64 2024-12-03T17:17:19,478 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/0d82c02158b2444a9062b261c7f2218d to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/0d82c02158b2444a9062b261c7f2218d 2024-12-03T17:17:19,480 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/5ba5cecdd0ac48bb82da7902a4464b49 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/5ba5cecdd0ac48bb82da7902a4464b49 2024-12-03T17:17:19,481 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/224558d47f3445399825040c07af8f28 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/224558d47f3445399825040c07af8f28 2024-12-03T17:17:19,483 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/b7a8585b0a5444b48a87dbf720f25c72 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/b7a8585b0a5444b48a87dbf720f25c72 2024-12-03T17:17:19,484 DEBUG [StoreCloser-TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/20beeaa50e5646fe89770f7d281b3ce8 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/20beeaa50e5646fe89770f7d281b3ce8 2024-12-03T17:17:19,491 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/recovered.edits/567.seqid, newMaxSeqId=567, maxSeqId=1 2024-12-03T17:17:19,492 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. 2024-12-03T17:17:19,492 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1676): Region close journal for f5a2a3f1964bd8ecfa7908952eaec708: Waiting for close lock at 1733246237855Running coprocessor pre-close hooks at 1733246237855Disabling compacts and flushes for region at 1733246237855Disabling writes for close at 1733246237855Obtaining lock to block concurrent updates at 1733246237855Preparing flush snapshotting stores in f5a2a3f1964bd8ecfa7908952eaec708 at 1733246237855Finished memstore snapshotting TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733246237856 (+1 ms)Flushing stores of TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708. at 1733246237863 (+7 ms)Flushing f5a2a3f1964bd8ecfa7908952eaec708/A: creating writer at 1733246237863Flushing f5a2a3f1964bd8ecfa7908952eaec708/A: appending metadata at 1733246237887 (+24 ms)Flushing f5a2a3f1964bd8ecfa7908952eaec708/A: closing flushed file at 1733246237888 (+1 ms)Flushing f5a2a3f1964bd8ecfa7908952eaec708/B: creating writer at 1733246238346 (+458 ms)Flushing f5a2a3f1964bd8ecfa7908952eaec708/B: appending metadata at 1733246238350 (+4 ms)Flushing f5a2a3f1964bd8ecfa7908952eaec708/B: closing flushed file at 1733246238350Flushing f5a2a3f1964bd8ecfa7908952eaec708/C: creating writer at 1733246238817 (+467 ms)Flushing f5a2a3f1964bd8ecfa7908952eaec708/C: appending metadata at 1733246238826 (+9 ms)Flushing f5a2a3f1964bd8ecfa7908952eaec708/C: closing flushed file at 1733246238826Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f6b32f7: reopening flushed file at 1733246238857 (+31 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@bf6e6ac: reopening flushed file at 1733246238865 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a925cd8: reopening flushed file at 1733246238872 (+7 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for f5a2a3f1964bd8ecfa7908952eaec708 in 1024ms, sequenceid=564, compaction requested=false at 1733246238879 (+7 ms)Writing region close event to WAL at 1733246239487 (+608 ms)Running coprocessor post-close hooks at 1733246239492 (+5 ms)Closed at 1733246239492 2024-12-03T17:17:19,495 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] handler.UnassignRegionHandler(157): Closed f5a2a3f1964bd8ecfa7908952eaec708 2024-12-03T17:17:19,496 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=f5a2a3f1964bd8ecfa7908952eaec708, regionState=CLOSED 2024-12-03T17:17:19,497 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=79, ppid=78, state=RUNNABLE, hasLock=false; CloseRegionProcedure f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 because future has completed 2024-12-03T17:17:19,502 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-12-03T17:17:19,502 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; CloseRegionProcedure f5a2a3f1964bd8ecfa7908952eaec708, server=27e0f852a164,41167,1733246151593 in 1.8020 sec 2024-12-03T17:17:19,504 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=78, resume processing ppid=77 2024-12-03T17:17:19,504 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, ppid=77, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f5a2a3f1964bd8ecfa7908952eaec708, UNASSIGN in 1.8140 sec 2024-12-03T17:17:19,506 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-12-03T17:17:19,507 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8190 sec 2024-12-03T17:17:19,508 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733246239508"}]},"ts":"1733246239508"} 2024-12-03T17:17:19,510 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-03T17:17:19,510 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-03T17:17:19,512 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8560 sec 2024-12-03T17:17:19,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-03T17:17:19,828 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:17:19,829 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-03T17:17:19,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T17:17:19,832 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=80, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T17:17:19,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-03T17:17:19,835 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=80, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T17:17:19,851 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708 2024-12-03T17:17:19,867 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A, FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B, FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C, FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/recovered.edits] 2024-12-03T17:17:19,906 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/4213fc1652e647ebaf3ebd2f78b504d3 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/4213fc1652e647ebaf3ebd2f78b504d3 2024-12-03T17:17:19,925 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/76fd2c8d536e414f96f62d0480a7edd2 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/A/76fd2c8d536e414f96f62d0480a7edd2 2024-12-03T17:17:19,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-03T17:17:19,963 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/4c0b18012da64953989673279c290ff0 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/4c0b18012da64953989673279c290ff0 2024-12-03T17:17:19,972 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/aa1e2335eeb44ffabcac342bb3e6f5b2 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/B/aa1e2335eeb44ffabcac342bb3e6f5b2 2024-12-03T17:17:19,977 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/49a730a52d354b6386dd60ef5b015b64 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/49a730a52d354b6386dd60ef5b015b64 2024-12-03T17:17:19,980 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/a62e3675ff0d4beb8ffd5b2b81ccf6b5 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/C/a62e3675ff0d4beb8ffd5b2b81ccf6b5 2024-12-03T17:17:19,990 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/recovered.edits/567.seqid to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708/recovered.edits/567.seqid 2024-12-03T17:17:19,996 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/f5a2a3f1964bd8ecfa7908952eaec708 2024-12-03T17:17:20,006 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-03T17:17:20,032 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=80, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T17:17:20,035 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-03T17:17:20,038 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-03T17:17:20,039 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=80, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T17:17:20,039 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-03T17:17:20,040 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733246240039"}]},"ts":"9223372036854775807"} 2024-12-03T17:17:20,042 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-03T17:17:20,042 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => f5a2a3f1964bd8ecfa7908952eaec708, NAME => 'TestAcidGuarantees,,1733246211530.f5a2a3f1964bd8ecfa7908952eaec708.', STARTKEY => '', ENDKEY => ''}] 2024-12-03T17:17:20,042 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-03T17:17:20,042 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733246240042"}]},"ts":"9223372036854775807"} 2024-12-03T17:17:20,045 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-03T17:17:20,046 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=80, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T17:17:20,047 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 217 msec 2024-12-03T17:17:20,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-03T17:17:20,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-03T17:17:20,155 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:17:20,169 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=249 (was 252), OpenFileDescriptor=455 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1444 (was 1527), ProcessCount=11 (was 11), AvailableMemoryMB=5005 (was 4560) - AvailableMemoryMB LEAK? - 2024-12-03T17:17:20,183 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=249, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=1444, ProcessCount=11, AvailableMemoryMB=5002 2024-12-03T17:17:20,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T17:17:20,185 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T17:17:20,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=81, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-03T17:17:20,188 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T17:17:20,188 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:17:20,189 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 81 2024-12-03T17:17:20,190 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T17:17:20,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-12-03T17:17:20,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742145_1321 (size=1061) 2024-12-03T17:17:20,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-12-03T17:17:20,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-12-03T17:17:20,607 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4 2024-12-03T17:17:20,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742146_1322 (size=53) 2024-12-03T17:17:20,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-12-03T17:17:21,029 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T17:17:21,029 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 86cf735dc0f15035813b7868e4f59179, disabling compactions & flushes 2024-12-03T17:17:21,029 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:21,029 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:21,029 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. after waiting 0 ms 2024-12-03T17:17:21,029 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:21,029 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:21,029 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 86cf735dc0f15035813b7868e4f59179: Waiting for close lock at 1733246241029Disabling compacts and flushes for region at 1733246241029Disabling writes for close at 1733246241029Writing region close event to WAL at 1733246241029Closed at 1733246241029 2024-12-03T17:17:21,031 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T17:17:21,031 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733246241031"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733246241031"}]},"ts":"1733246241031"} 2024-12-03T17:17:21,033 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T17:17:21,035 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T17:17:21,035 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733246241035"}]},"ts":"1733246241035"} 2024-12-03T17:17:21,037 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-03T17:17:21,037 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=86cf735dc0f15035813b7868e4f59179, ASSIGN}] 2024-12-03T17:17:21,039 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=86cf735dc0f15035813b7868e4f59179, ASSIGN 2024-12-03T17:17:21,041 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=86cf735dc0f15035813b7868e4f59179, ASSIGN; state=OFFLINE, location=27e0f852a164,41167,1733246151593; forceNewPlan=false, retain=false 2024-12-03T17:17:21,122 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-03T17:17:21,191 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=86cf735dc0f15035813b7868e4f59179, regionState=OPENING, regionLocation=27e0f852a164,41167,1733246151593 2024-12-03T17:17:21,193 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=86cf735dc0f15035813b7868e4f59179, ASSIGN because future has completed 2024-12-03T17:17:21,194 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; OpenRegionProcedure 86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593}] 2024-12-03T17:17:21,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-12-03T17:17:21,349 INFO [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:21,350 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7752): Opening region: {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} 2024-12-03T17:17:21,350 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:21,350 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T17:17:21,350 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7794): checking encryption for 86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:21,350 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7797): checking classloading for 86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:21,352 INFO [StoreOpener-86cf735dc0f15035813b7868e4f59179-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:21,353 INFO [StoreOpener-86cf735dc0f15035813b7868e4f59179-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T17:17:21,353 INFO [StoreOpener-86cf735dc0f15035813b7868e4f59179-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 86cf735dc0f15035813b7868e4f59179 columnFamilyName A 2024-12-03T17:17:21,353 DEBUG [StoreOpener-86cf735dc0f15035813b7868e4f59179-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:17:21,354 INFO [StoreOpener-86cf735dc0f15035813b7868e4f59179-1 {}] regionserver.HStore(327): Store=86cf735dc0f15035813b7868e4f59179/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T17:17:21,354 INFO [StoreOpener-86cf735dc0f15035813b7868e4f59179-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:21,355 INFO [StoreOpener-86cf735dc0f15035813b7868e4f59179-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T17:17:21,355 INFO [StoreOpener-86cf735dc0f15035813b7868e4f59179-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 86cf735dc0f15035813b7868e4f59179 columnFamilyName B 2024-12-03T17:17:21,355 DEBUG [StoreOpener-86cf735dc0f15035813b7868e4f59179-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:17:21,356 INFO [StoreOpener-86cf735dc0f15035813b7868e4f59179-1 {}] regionserver.HStore(327): Store=86cf735dc0f15035813b7868e4f59179/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T17:17:21,356 INFO [StoreOpener-86cf735dc0f15035813b7868e4f59179-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:21,357 INFO [StoreOpener-86cf735dc0f15035813b7868e4f59179-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T17:17:21,357 INFO [StoreOpener-86cf735dc0f15035813b7868e4f59179-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 86cf735dc0f15035813b7868e4f59179 columnFamilyName C 2024-12-03T17:17:21,357 DEBUG [StoreOpener-86cf735dc0f15035813b7868e4f59179-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:17:21,357 INFO [StoreOpener-86cf735dc0f15035813b7868e4f59179-1 {}] regionserver.HStore(327): Store=86cf735dc0f15035813b7868e4f59179/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T17:17:21,357 INFO [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:21,358 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1038): replaying wal for 86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:21,358 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:21,359 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:21,360 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1048): stopping wal replay for 86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:21,360 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1060): Cleaning up temporary data for 86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:21,360 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T17:17:21,361 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1093): writing seq id for 86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:21,363 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T17:17:21,363 INFO [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1114): Opened 86cf735dc0f15035813b7868e4f59179; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58946760, jitterRate=-0.12162482738494873}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T17:17:21,363 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:21,364 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1006): Region open journal for 86cf735dc0f15035813b7868e4f59179: Running coprocessor pre-open hook at 1733246241350Writing region info on filesystem at 1733246241350Initializing all the Stores at 1733246241351 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733246241351Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733246241352 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733246241352Cleaning up temporary data from old regions at 1733246241360 (+8 ms)Running coprocessor post-open hooks at 1733246241363 (+3 ms)Region opened successfully at 1733246241364 (+1 ms) 2024-12-03T17:17:21,364 INFO [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., pid=83, masterSystemTime=1733246241346 2024-12-03T17:17:21,367 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:21,367 INFO [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:21,367 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=86cf735dc0f15035813b7868e4f59179, regionState=OPEN, openSeqNum=2, regionLocation=27e0f852a164,41167,1733246151593 2024-12-03T17:17:21,369 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE, hasLock=false; OpenRegionProcedure 86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 because future has completed 2024-12-03T17:17:21,371 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-12-03T17:17:21,371 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; OpenRegionProcedure 86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 in 176 msec 2024-12-03T17:17:21,373 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-12-03T17:17:21,373 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=86cf735dc0f15035813b7868e4f59179, ASSIGN in 334 msec 2024-12-03T17:17:21,374 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T17:17:21,374 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733246241374"}]},"ts":"1733246241374"} 2024-12-03T17:17:21,376 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-03T17:17:21,377 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T17:17:21,378 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1920 sec 2024-12-03T17:17:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-12-03T17:17:22,334 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:17:22,335 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ec1958f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:22,335 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:17:22,335 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:17:22,339 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:17:22,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:17:22,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:17:22,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@247a60b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:22,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:17:22,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:17:22,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:22,343 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54078, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:17:22,344 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68925e6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:22,344 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:17:22,346 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:17:22,347 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:17:22,351 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51230, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:17:22,352 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T17:17:22,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 27e0f852a164,38523,1733246150595 2024-12-03T17:17:22,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7fe492ab 2024-12-03T17:17:22,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T17:17:22,355 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54086, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T17:17:22,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T17:17:22,358 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T17:17:22,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-03T17:17:22,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742147_1323 (size=1097) 2024-12-03T17:17:22,792 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-03T17:17:22,792 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-03T17:17:22,796 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T17:17:22,800 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=86cf735dc0f15035813b7868e4f59179, REOPEN/MOVE}] 2024-12-03T17:17:22,801 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=86cf735dc0f15035813b7868e4f59179, REOPEN/MOVE 2024-12-03T17:17:22,802 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=86cf735dc0f15035813b7868e4f59179, regionState=CLOSING, regionLocation=27e0f852a164,41167,1733246151593 2024-12-03T17:17:22,807 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=86cf735dc0f15035813b7868e4f59179, REOPEN/MOVE because future has completed 2024-12-03T17:17:22,807 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T17:17:22,807 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure 86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593}] 2024-12-03T17:17:22,961 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(122): Close 86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:22,961 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T17:17:22,961 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1722): Closing 86cf735dc0f15035813b7868e4f59179, disabling compactions & flushes 2024-12-03T17:17:22,961 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:22,961 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:22,961 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. after waiting 0 ms 2024-12-03T17:17:22,961 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:22,966 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T17:17:22,967 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:22,967 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1676): Region close journal for 86cf735dc0f15035813b7868e4f59179: Waiting for close lock at 1733246242961Running coprocessor pre-close hooks at 1733246242961Disabling compacts and flushes for region at 1733246242961Disabling writes for close at 1733246242961Writing region close event to WAL at 1733246242962 (+1 ms)Running coprocessor post-close hooks at 1733246242967 (+5 ms)Closed at 1733246242967 2024-12-03T17:17:22,967 WARN [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionServer(3299): Not adding moved region record: 86cf735dc0f15035813b7868e4f59179 to self. 2024-12-03T17:17:22,970 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(157): Closed 86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:22,971 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=86cf735dc0f15035813b7868e4f59179, regionState=CLOSED 2024-12-03T17:17:22,972 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure 86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 because future has completed 2024-12-03T17:17:22,975 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-12-03T17:17:22,975 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseRegionProcedure 86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 in 166 msec 2024-12-03T17:17:22,976 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=86cf735dc0f15035813b7868e4f59179, REOPEN/MOVE; state=CLOSED, location=27e0f852a164,41167,1733246151593; forceNewPlan=false, retain=true 2024-12-03T17:17:23,127 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=86cf735dc0f15035813b7868e4f59179, regionState=OPENING, regionLocation=27e0f852a164,41167,1733246151593 2024-12-03T17:17:23,129 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=86cf735dc0f15035813b7868e4f59179, REOPEN/MOVE because future has completed 2024-12-03T17:17:23,129 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593}] 2024-12-03T17:17:23,284 INFO [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:23,285 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7752): Opening region: {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} 2024-12-03T17:17:23,285 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:23,285 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T17:17:23,285 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7794): checking encryption for 86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:23,285 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7797): checking classloading for 86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:23,286 INFO [StoreOpener-86cf735dc0f15035813b7868e4f59179-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:23,287 INFO [StoreOpener-86cf735dc0f15035813b7868e4f59179-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T17:17:23,287 INFO [StoreOpener-86cf735dc0f15035813b7868e4f59179-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 86cf735dc0f15035813b7868e4f59179 columnFamilyName A 2024-12-03T17:17:23,289 DEBUG [StoreOpener-86cf735dc0f15035813b7868e4f59179-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:17:23,289 INFO [StoreOpener-86cf735dc0f15035813b7868e4f59179-1 {}] regionserver.HStore(327): Store=86cf735dc0f15035813b7868e4f59179/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T17:17:23,289 INFO [StoreOpener-86cf735dc0f15035813b7868e4f59179-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:23,290 INFO [StoreOpener-86cf735dc0f15035813b7868e4f59179-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T17:17:23,290 INFO [StoreOpener-86cf735dc0f15035813b7868e4f59179-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 86cf735dc0f15035813b7868e4f59179 columnFamilyName B 2024-12-03T17:17:23,290 DEBUG [StoreOpener-86cf735dc0f15035813b7868e4f59179-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:17:23,291 INFO [StoreOpener-86cf735dc0f15035813b7868e4f59179-1 {}] regionserver.HStore(327): Store=86cf735dc0f15035813b7868e4f59179/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T17:17:23,291 INFO [StoreOpener-86cf735dc0f15035813b7868e4f59179-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:23,291 INFO [StoreOpener-86cf735dc0f15035813b7868e4f59179-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T17:17:23,291 INFO [StoreOpener-86cf735dc0f15035813b7868e4f59179-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 86cf735dc0f15035813b7868e4f59179 columnFamilyName C 2024-12-03T17:17:23,291 DEBUG [StoreOpener-86cf735dc0f15035813b7868e4f59179-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:17:23,292 INFO [StoreOpener-86cf735dc0f15035813b7868e4f59179-1 {}] regionserver.HStore(327): Store=86cf735dc0f15035813b7868e4f59179/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T17:17:23,292 INFO [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:23,292 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1038): replaying wal for 86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:23,293 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:23,294 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:23,295 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1048): stopping wal replay for 86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:23,295 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1060): Cleaning up temporary data for 86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:23,295 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T17:17:23,296 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1093): writing seq id for 86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:23,297 INFO [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1114): Opened 86cf735dc0f15035813b7868e4f59179; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71655256, jitterRate=0.06774652004241943}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T17:17:23,297 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:23,298 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1006): Region open journal for 86cf735dc0f15035813b7868e4f59179: Running coprocessor pre-open hook at 1733246243285Writing region info on filesystem at 1733246243285Initializing all the Stores at 1733246243286 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733246243286Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733246243286Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733246243286Cleaning up temporary data from old regions at 1733246243295 (+9 ms)Running coprocessor post-open hooks at 1733246243297 (+2 ms)Region opened successfully at 1733246243298 (+1 ms) 2024-12-03T17:17:23,298 INFO [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., pid=88, masterSystemTime=1733246243281 2024-12-03T17:17:23,300 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:23,301 INFO [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:23,301 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=86cf735dc0f15035813b7868e4f59179, regionState=OPEN, openSeqNum=5, regionLocation=27e0f852a164,41167,1733246151593 2024-12-03T17:17:23,302 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 because future has completed 2024-12-03T17:17:23,305 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=86 2024-12-03T17:17:23,305 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=86, state=SUCCESS, hasLock=false; OpenRegionProcedure 86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 in 174 msec 2024-12-03T17:17:23,307 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-12-03T17:17:23,307 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=86cf735dc0f15035813b7868e4f59179, REOPEN/MOVE in 505 msec 2024-12-03T17:17:23,310 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-12-03T17:17:23,310 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 512 msec 2024-12-03T17:17:23,312 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 952 msec 2024-12-03T17:17:23,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-03T17:17:23,313 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:17:23,314 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4600746, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:23,314 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:17:23,314 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:17:23,315 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:17:23,315 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:17:23,315 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:17:23,316 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@395a98e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:23,316 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:17:23,316 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:17:23,316 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:23,317 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54096, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:17:23,318 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68a0d48a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:23,318 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6447def3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:23,319 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:17:23,319 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:17:23,320 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:17:23,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:17:23,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:17:23,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fac0500, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:23,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:17:23,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:17:23,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:23,321 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54114, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:17:23,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6125d010, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:23,323 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5345722e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:23,323 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:17:23,323 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:17:23,324 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:17:23,324 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:17:23,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:17:23,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30625ec5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:23,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:17:23,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:17:23,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:23,326 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54142, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:17:23,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@698425ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:23,330 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dd3a90f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:23,330 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:17:23,330 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:17:23,331 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:17:23,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:17:23,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:17:23,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cd06586, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:23,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:17:23,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:17:23,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:23,333 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54164, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:17:23,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@722df4db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:23,335 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10e2b5a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:23,335 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:17:23,335 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:17:23,336 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:17:23,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:17:23,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:17:23,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36e3e925, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:23,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:17:23,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:17:23,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:23,338 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54184, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:17:23,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ba416cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:23,339 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33f8ca2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:23,339 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:17:23,339 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:17:23,340 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:17:23,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:17:23,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:17:23,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f04188f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:23,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:17:23,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:17:23,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:23,341 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54194, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:17:23,342 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24098e77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:23,342 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1515ed5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:23,342 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:17:23,343 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:17:23,343 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:17:23,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:17:23,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:17:23,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ada85cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:23,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:17:23,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:17:23,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:23,345 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54218, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:17:23,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e8ca1ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:23,346 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d10c554, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:23,346 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:17:23,346 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:17:23,347 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:17:23,347 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:17:23,347 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:17:23,347 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23b07675, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:23,347 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:17:23,347 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:17:23,347 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:23,348 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54236, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:17:23,349 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2645bd8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:23,350 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26b7b919, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:23,350 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:17:23,351 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:17:23,352 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:17:23,352 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:17:23,352 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:17:23,352 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@213b285, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:23,352 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:17:23,352 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:17:23,353 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:23,353 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54260, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:17:23,354 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a21100e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:23,355 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dacd8f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:23,355 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:17:23,355 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:17:23,356 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:17:23,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:17:23,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:17:23,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@775591b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:23,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:17:23,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:17:23,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:23,358 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54272, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:17:23,358 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20e1c6e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:23,359 DEBUG [Thread-1485 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:17:23,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:17:23,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:17:23,361 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51234, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:17:23,363 DEBUG [Thread-1492 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:17:23,363 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T17:17:23,363 DEBUG [Thread-1487 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:17:23,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5] 2024-12-03T17:17:23,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees 2024-12-03T17:17:23,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:17:23,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:17:23,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-03T17:17:23,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:17:23,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:17:23,366 DEBUG [Thread-1489 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:17:23,366 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T17:17:23,367 DEBUG [Thread-1498 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:17:23,367 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51238, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:17:23,367 DEBUG [Thread-1496 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:17:23,367 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51244, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:17:23,367 DEBUG [Thread-1500 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:17:23,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:17:23,368 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:17:23,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:17:23,368 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:17:23,368 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T17:17:23,368 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T17:17:23,369 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51252, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:17:23,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5] 2024-12-03T17:17:23,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5] 2024-12-03T17:17:23,372 DEBUG [Thread-1496 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:17:23,372 DEBUG [Thread-1483 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:17:23,372 DEBUG [Thread-1496 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:17:23,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:17:23,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:17:23,373 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51262, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:17:23,373 DEBUG [Thread-1481 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:17:23,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5] 2024-12-03T17:17:23,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5] 2024-12-03T17:17:23,375 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51286, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:17:23,376 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:17:23,376 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:17:23,376 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:17:23,376 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:17:23,376 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51276, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:17:23,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5] 2024-12-03T17:17:23,377 DEBUG [Thread-1494 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:17:23,377 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:17:23,378 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:17:23,378 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51294, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:17:23,380 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51304, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:17:23,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5] 2024-12-03T17:17:23,380 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51312, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:17:23,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5] 2024-12-03T17:17:23,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5] 2024-12-03T17:17:23,384 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5] 2024-12-03T17:17:23,384 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86cf735dc0f15035813b7868e4f59179 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T17:17:23,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:23,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=A 2024-12-03T17:17:23,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:23,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=B 2024-12-03T17:17:23,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:23,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=C 2024-12-03T17:17:23,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:23,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203255acc78f89b466895bbfc8838489b44_86cf735dc0f15035813b7868e4f59179 is 50, key is test_row_0/A:col10/1733246243384/Put/seqid=0 2024-12-03T17:17:23,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742148_1324 (size=12154) 2024-12-03T17:17:23,438 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:17:23,446 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203255acc78f89b466895bbfc8838489b44_86cf735dc0f15035813b7868e4f59179 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203255acc78f89b466895bbfc8838489b44_86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:23,448 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/48e4f5cae8f848a9aa711765b4bd4184, store: [table=TestAcidGuarantees family=A region=86cf735dc0f15035813b7868e4f59179] 2024-12-03T17:17:23,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/48e4f5cae8f848a9aa711765b4bd4184 is 175, key is test_row_0/A:col10/1733246243384/Put/seqid=0 2024-12-03T17:17:23,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:23,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51294 deadline: 1733246303427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:23,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:23,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:23,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:23,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:23,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51304 deadline: 1733246303436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:23,459 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:23,459 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:23,459 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:23,461 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733246243423, max timestamp=1733246243449Num uniques 859; 2024-12-03T17:17:23,462 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T17:17:23,465 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733246243423, max timestamp=1733246243449Num uniques 859; 2024-12-03T17:17:23,465 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T17:17:23,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742149_1325 (size=30955) 2024-12-03T17:17:23,471 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/48e4f5cae8f848a9aa711765b4bd4184 2024-12-03T17:17:23,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-03T17:17:23,476 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733246243423, max timestamp=1733246243449Num uniques 859; 2024-12-03T17:17:23,477 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T17:17:23,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/c320b220742c4ae7937322c33eda32a7 is 50, key is test_row_0/B:col10/1733246243384/Put/seqid=0 2024-12-03T17:17:23,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-03T17:17:23,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:23,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. as already flushing 2024-12-03T17:17:23,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:23,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:23,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:23,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:23,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742150_1326 (size=12001) 2024-12-03T17:17:23,564 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/c320b220742c4ae7937322c33eda32a7 2024-12-03T17:17:23,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:23,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246303548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:23,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:23,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:23,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:23,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:23,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246303551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:23,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:23,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246303551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:23,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:23,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:23,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:23,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:23,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51294 deadline: 1733246303564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:23,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:23,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:23,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:23,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:23,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51304 deadline: 1733246303566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:23,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:23,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:23,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:23,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:23,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:23,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:23,600 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/97ccc85f920d49a085a0acf4e0585131 is 50, key is test_row_0/C:col10/1733246243384/Put/seqid=0 2024-12-03T17:17:23,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742151_1327 (size=12001) 2024-12-03T17:17:23,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-03T17:17:23,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:23,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. as already flushing 2024-12-03T17:17:23,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:23,677 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:23,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:23,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:23,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-03T17:17:23,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:23,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246303674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:23,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:23,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:23,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:23,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:23,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246303683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:23,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:23,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:23,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:23,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:23,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246303683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:23,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:23,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:23,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:23,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:23,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51294 deadline: 1733246303794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:23,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:23,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:23,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:23,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:23,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51304 deadline: 1733246303794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:23,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:23,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:23,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:23,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-03T17:17:23,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:23,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. as already flushing 2024-12-03T17:17:23,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:23,830 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:23,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:23,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:23,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:23,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:23,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246303894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:23,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246303894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:23,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:23,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:23,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:23,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:23,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:23,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:23,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246303895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:23,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:23,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:23,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:23,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:23,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-03T17:17:23,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:23,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. as already flushing 2024-12-03T17:17:23,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:23,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:23,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:23,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:23,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-03T17:17:24,025 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/97ccc85f920d49a085a0acf4e0585131 2024-12-03T17:17:24,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/48e4f5cae8f848a9aa711765b4bd4184 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/48e4f5cae8f848a9aa711765b4bd4184 2024-12-03T17:17:24,074 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/48e4f5cae8f848a9aa711765b4bd4184, entries=150, sequenceid=16, filesize=30.2 K 2024-12-03T17:17:24,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/c320b220742c4ae7937322c33eda32a7 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/c320b220742c4ae7937322c33eda32a7 2024-12-03T17:17:24,126 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/c320b220742c4ae7937322c33eda32a7, entries=150, sequenceid=16, filesize=11.7 K 2024-12-03T17:17:24,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:24,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51294 deadline: 1733246304114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:24,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:24,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51304 deadline: 1733246304115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:24,128 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:24,128 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:24,128 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:24,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:24,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:24,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:24,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/97ccc85f920d49a085a0acf4e0585131 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/97ccc85f920d49a085a0acf4e0585131 2024-12-03T17:17:24,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-03T17:17:24,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:24,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. as already flushing 2024-12-03T17:17:24,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:24,143 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:24,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:24,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:24,184 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/97ccc85f920d49a085a0acf4e0585131, entries=150, sequenceid=16, filesize=11.7 K 2024-12-03T17:17:24,188 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=134.18 KB/137400 for 86cf735dc0f15035813b7868e4f59179 in 804ms, sequenceid=16, compaction requested=false 2024-12-03T17:17:24,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86cf735dc0f15035813b7868e4f59179: 2024-12-03T17:17:24,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:24,236 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86cf735dc0f15035813b7868e4f59179 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-03T17:17:24,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=A 2024-12-03T17:17:24,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T17:17:24,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=B 2024-12-03T17:17:24,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T17:17:24,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=C 2024-12-03T17:17:24,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T17:17:24,276 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412037cb3f3ac05e44bf4826fc5320bfee668_86cf735dc0f15035813b7868e4f59179 is 50, key is test_row_0/A:col10/1733246243516/Put/seqid=0 2024-12-03T17:17:24,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:24,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246304274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:24,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:24,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246304277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:24,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:24,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:24,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:24,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:24,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:24,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:24,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:24,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246304278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:24,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:24,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:24,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:24,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-03T17:17:24,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742152_1328 (size=12154) 2024-12-03T17:17:24,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:24,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. as already flushing 2024-12-03T17:17:24,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:24,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:24,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:24,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:24,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:24,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246304400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:24,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:24,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:24,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:24,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:24,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246304402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:24,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:24,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246304404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:24,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:24,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:24,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:24,418 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:24,418 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:24,418 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:24,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-03T17:17:24,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:24,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. as already flushing 2024-12-03T17:17:24,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:24,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:24,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:24,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:24,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-03T17:17:24,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-03T17:17:24,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:24,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. as already flushing 2024-12-03T17:17:24,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:24,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:24,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:24,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:24,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246304618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:24,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:24,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:24,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:24,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:24,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:24,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246304624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:24,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:24,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:24,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:24,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246304624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:24,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:24,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:24,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:24,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:24,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:24,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51294 deadline: 1733246304634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:24,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:24,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:24,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:24,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51304 deadline: 1733246304634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:24,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:24,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:24,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:24,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:24,714 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:17:24,724 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412037cb3f3ac05e44bf4826fc5320bfee668_86cf735dc0f15035813b7868e4f59179 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412037cb3f3ac05e44bf4826fc5320bfee668_86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:24,725 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/80b8a8f2d55041d5af075f0e947512e1, store: [table=TestAcidGuarantees family=A region=86cf735dc0f15035813b7868e4f59179] 2024-12-03T17:17:24,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/80b8a8f2d55041d5af075f0e947512e1 is 175, key is test_row_0/A:col10/1733246243516/Put/seqid=0 2024-12-03T17:17:24,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742153_1329 (size=30955) 2024-12-03T17:17:24,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-03T17:17:24,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:24,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. as already flushing 2024-12-03T17:17:24,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:24,781 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:24,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:24,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:24,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-03T17:17:24,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:24,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. as already flushing 2024-12-03T17:17:24,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:24,939 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:24,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:24,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:24,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:24,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246304933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:24,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:24,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:24,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:24,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:24,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246304944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:24,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:24,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:24,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:24,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:24,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246304945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:24,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:24,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:24,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:25,038 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T17:17:25,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-03T17:17:25,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:25,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. as already flushing 2024-12-03T17:17:25,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:25,096 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:25,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:25,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:25,151 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/80b8a8f2d55041d5af075f0e947512e1 2024-12-03T17:17:25,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/4cf4ff6088a245deadaf3f3df91b5c03 is 50, key is test_row_0/B:col10/1733246243516/Put/seqid=0 2024-12-03T17:17:25,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742154_1330 (size=12001) 2024-12-03T17:17:25,254 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-03T17:17:25,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:25,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. as already flushing 2024-12-03T17:17:25,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:25,254 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:25,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:25,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:25,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-03T17:17:25,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:25,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. as already flushing 2024-12-03T17:17:25,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:25,417 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:25,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:25,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:25,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:25,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246305464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:25,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:25,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:25,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:25,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:25,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246305464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:25,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:25,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:25,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:25,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:25,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246305473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:25,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:25,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:25,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:25,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-03T17:17:25,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-03T17:17:25,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:25,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. as already flushing 2024-12-03T17:17:25,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:25,575 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:25,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:25,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:25,605 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/4cf4ff6088a245deadaf3f3df91b5c03 2024-12-03T17:17:25,613 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/eae44ee6df8744ee8e77cec1545cc69d is 50, key is test_row_0/C:col10/1733246243516/Put/seqid=0 2024-12-03T17:17:25,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742155_1331 (size=12001) 2024-12-03T17:17:25,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:25,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51294 deadline: 1733246305654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:25,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:25,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51304 deadline: 1733246305654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:25,667 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 27e0f852a164,41167,1733246151593 for 'test_row_2' in 86cf735dc0f15035813b7868e4f59179 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2240 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T17:17:25,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:25,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:25,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:25,668 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 27e0f852a164,41167,1733246151593 for 'test_row_2' in 86cf735dc0f15035813b7868e4f59179 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2231 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T17:17:25,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:25,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:25,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:25,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-03T17:17:25,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:25,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. as already flushing 2024-12-03T17:17:25,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:25,730 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:25,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:25,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:25,886 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-03T17:17:25,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:25,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. as already flushing 2024-12-03T17:17:25,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:25,886 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:25,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:25,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:26,025 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/eae44ee6df8744ee8e77cec1545cc69d 2024-12-03T17:17:26,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/80b8a8f2d55041d5af075f0e947512e1 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/80b8a8f2d55041d5af075f0e947512e1 2024-12-03T17:17:26,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-03T17:17:26,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:26,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. as already flushing 2024-12-03T17:17:26,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:26,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:26,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:26,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:26,053 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/80b8a8f2d55041d5af075f0e947512e1, entries=150, sequenceid=54, filesize=30.2 K 2024-12-03T17:17:26,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/4cf4ff6088a245deadaf3f3df91b5c03 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/4cf4ff6088a245deadaf3f3df91b5c03 2024-12-03T17:17:26,062 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/4cf4ff6088a245deadaf3f3df91b5c03, entries=150, sequenceid=54, filesize=11.7 K 2024-12-03T17:17:26,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/eae44ee6df8744ee8e77cec1545cc69d as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/eae44ee6df8744ee8e77cec1545cc69d 2024-12-03T17:17:26,067 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/eae44ee6df8744ee8e77cec1545cc69d, entries=150, sequenceid=54, filesize=11.7 K 2024-12-03T17:17:26,070 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=33.54 KB/34350 for 86cf735dc0f15035813b7868e4f59179 in 1834ms, sequenceid=54, compaction requested=false 2024-12-03T17:17:26,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86cf735dc0f15035813b7868e4f59179: 2024-12-03T17:17:26,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-03T17:17:26,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:26,204 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2902): Flushing 86cf735dc0f15035813b7868e4f59179 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-03T17:17:26,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=A 2024-12-03T17:17:26,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:26,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=B 2024-12-03T17:17:26,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:26,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=C 2024-12-03T17:17:26,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:26,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203047c06021eca4582a07092b58d24b5c9_86cf735dc0f15035813b7868e4f59179 is 50, key is test_row_0/A:col10/1733246244249/Put/seqid=0 2024-12-03T17:17:26,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742156_1332 (size=12154) 2024-12-03T17:17:26,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:17:26,257 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203047c06021eca4582a07092b58d24b5c9_86cf735dc0f15035813b7868e4f59179 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203047c06021eca4582a07092b58d24b5c9_86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:26,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/a0bc32c6219f4f7eb594eeff575375af, store: [table=TestAcidGuarantees family=A region=86cf735dc0f15035813b7868e4f59179] 2024-12-03T17:17:26,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/a0bc32c6219f4f7eb594eeff575375af is 175, key is test_row_0/A:col10/1733246244249/Put/seqid=0 2024-12-03T17:17:26,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742157_1333 (size=30955) 2024-12-03T17:17:26,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:26,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. as already flushing 2024-12-03T17:17:26,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733246246511, max timestamp=1733246246548Num uniques 859; 2024-12-03T17:17:26,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T17:17:26,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:26,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246306582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:26,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:26,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:26,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246306585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:26,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:26,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:26,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:26,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:26,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:26,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:26,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246306611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:26,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:26,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:26,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:26,675 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/a0bc32c6219f4f7eb594eeff575375af 2024-12-03T17:17:26,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/fac0875a75ed4c7a91cc6561917b2be8 is 50, key is test_row_0/B:col10/1733246244249/Put/seqid=0 2024-12-03T17:17:26,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742158_1334 (size=12001) 2024-12-03T17:17:26,731 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/fac0875a75ed4c7a91cc6561917b2be8 2024-12-03T17:17:26,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:26,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246306715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:26,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:26,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:26,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:26,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:26,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246306723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:26,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:26,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:26,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:26,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/89c68435e1cf473492410d7aaf6dff79 is 50, key is test_row_0/C:col10/1733246244249/Put/seqid=0 2024-12-03T17:17:26,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742159_1335 (size=12001) 2024-12-03T17:17:26,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:26,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246306744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:26,763 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:26,763 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:26,763 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:26,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:26,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246306944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:26,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:26,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246306944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:26,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:26,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:26,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:26,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:26,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:26,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:26,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:26,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246306973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:26,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:26,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:26,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:27,161 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/89c68435e1cf473492410d7aaf6dff79 2024-12-03T17:17:27,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/a0bc32c6219f4f7eb594eeff575375af as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/a0bc32c6219f4f7eb594eeff575375af 2024-12-03T17:17:27,185 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/a0bc32c6219f4f7eb594eeff575375af, entries=150, sequenceid=62, filesize=30.2 K 2024-12-03T17:17:27,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/fac0875a75ed4c7a91cc6561917b2be8 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/fac0875a75ed4c7a91cc6561917b2be8 2024-12-03T17:17:27,192 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/fac0875a75ed4c7a91cc6561917b2be8, entries=150, sequenceid=62, filesize=11.7 K 2024-12-03T17:17:27,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/89c68435e1cf473492410d7aaf6dff79 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/89c68435e1cf473492410d7aaf6dff79 2024-12-03T17:17:27,197 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/89c68435e1cf473492410d7aaf6dff79, entries=150, sequenceid=62, filesize=11.7 K 2024-12-03T17:17:27,201 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=145.36 KB/148850 for 86cf735dc0f15035813b7868e4f59179 in 997ms, sequenceid=62, compaction requested=true 2024-12-03T17:17:27,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2603): Flush status journal for 86cf735dc0f15035813b7868e4f59179: 2024-12-03T17:17:27,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:27,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=90 2024-12-03T17:17:27,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4169): Remote procedure done, pid=90 2024-12-03T17:17:27,206 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-12-03T17:17:27,206 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.8360 sec 2024-12-03T17:17:27,208 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees in 3.8440 sec 2024-12-03T17:17:27,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:27,282 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86cf735dc0f15035813b7868e4f59179 3/3 column families, dataSize=152.07 KB heapSize=446.48 KB 2024-12-03T17:17:27,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=A 2024-12-03T17:17:27,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:17:27,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=B 2024-12-03T17:17:27,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:17:27,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=C 2024-12-03T17:17:27,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:17:27,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203bc3cefd92653446b9baeba610c2fef99_86cf735dc0f15035813b7868e4f59179 is 50, key is test_row_0/A:col10/1733246247281/Put/seqid=0 2024-12-03T17:17:27,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:27,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246307307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:27,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:27,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246307309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:27,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:27,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:27,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:27,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:27,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:27,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:27,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742160_1336 (size=17034) 2024-12-03T17:17:27,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:27,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246307331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:27,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:27,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:27,347 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:27,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:27,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246307434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:27,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:27,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246307434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:27,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:27,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:27,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:27,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:27,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:27,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:27,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:27,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246307454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:27,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:27,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:27,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:27,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-03T17:17:27,532 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:17:27,533 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T17:17:27,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-12-03T17:17:27,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-03T17:17:27,536 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T17:17:27,537 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T17:17:27,537 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T17:17:27,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-03T17:17:27,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:27,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246307655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:27,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-03T17:17:27,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:27,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. as already flushing 2024-12-03T17:17:27,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:27,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:27,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:27,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:27,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246307655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:27,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:27,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:27,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:27,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:27,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:27,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:27,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:27,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:27,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51294 deadline: 1733246307707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:27,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:27,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246307707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:27,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:27,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51304 deadline: 1733246307708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:27,713 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 27e0f852a164,41167,1733246151593 for 'test_row_2' in 86cf735dc0f15035813b7868e4f59179 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4277 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T17:17:27,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:27,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:27,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:27,718 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 27e0f852a164,41167,1733246151593 for 'test_row_2' in 86cf735dc0f15035813b7868e4f59179 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4292 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T17:17:27,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:27,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:27,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:27,721 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:27,721 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:27,721 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:27,742 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:17:27,771 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203bc3cefd92653446b9baeba610c2fef99_86cf735dc0f15035813b7868e4f59179 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203bc3cefd92653446b9baeba610c2fef99_86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:27,778 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/04ec9678e612435886d5866904f507ff, store: [table=TestAcidGuarantees family=A region=86cf735dc0f15035813b7868e4f59179] 2024-12-03T17:17:27,778 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/04ec9678e612435886d5866904f507ff is 175, key is test_row_0/A:col10/1733246247281/Put/seqid=0 2024-12-03T17:17:27,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742161_1337 (size=48139) 2024-12-03T17:17:27,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-03T17:17:27,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-03T17:17:27,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:27,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. as already flushing 2024-12-03T17:17:27,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:27,863 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:27,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:27,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:28,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:28,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-03T17:17:28,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246308014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:28,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:28,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. as already flushing 2024-12-03T17:17:28,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:28,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:28,023 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:28,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:28,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:28,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:28,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:28,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:28,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246308018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:28,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:28,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:28,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:28,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:28,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246308034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:28,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:28,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:28,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:28,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-03T17:17:28,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-03T17:17:28,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:28,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. as already flushing 2024-12-03T17:17:28,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:28,179 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:28,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:28,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:28,216 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=93, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/04ec9678e612435886d5866904f507ff 2024-12-03T17:17:28,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/954c951e23214cfe908a22934920f6bc is 50, key is test_row_0/B:col10/1733246247281/Put/seqid=0 2024-12-03T17:17:28,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742162_1338 (size=12001) 2024-12-03T17:17:28,291 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/954c951e23214cfe908a22934920f6bc 2024-12-03T17:17:28,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/08690ce989124676ad79c8de07ec0e41 is 50, key is test_row_0/C:col10/1733246247281/Put/seqid=0 2024-12-03T17:17:28,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742163_1339 (size=12001) 2024-12-03T17:17:28,329 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/08690ce989124676ad79c8de07ec0e41 2024-12-03T17:17:28,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/04ec9678e612435886d5866904f507ff as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/04ec9678e612435886d5866904f507ff 2024-12-03T17:17:28,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-03T17:17:28,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:28,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. as already flushing 2024-12-03T17:17:28,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:28,338 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:28,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:28,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:28,342 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/04ec9678e612435886d5866904f507ff, entries=250, sequenceid=93, filesize=47.0 K 2024-12-03T17:17:28,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/954c951e23214cfe908a22934920f6bc as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/954c951e23214cfe908a22934920f6bc 2024-12-03T17:17:28,348 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/954c951e23214cfe908a22934920f6bc, entries=150, sequenceid=93, filesize=11.7 K 2024-12-03T17:17:28,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/08690ce989124676ad79c8de07ec0e41 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/08690ce989124676ad79c8de07ec0e41 2024-12-03T17:17:28,354 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/08690ce989124676ad79c8de07ec0e41, entries=150, sequenceid=93, filesize=11.7 K 2024-12-03T17:17:28,355 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~158.78 KB/162590, heapSize ~466.36 KB/477552, currentSize=20.13 KB/20610 for 86cf735dc0f15035813b7868e4f59179 in 1073ms, sequenceid=93, compaction requested=true 2024-12-03T17:17:28,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86cf735dc0f15035813b7868e4f59179: 2024-12-03T17:17:28,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86cf735dc0f15035813b7868e4f59179:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T17:17:28,356 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T17:17:28,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:28,356 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T17:17:28,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86cf735dc0f15035813b7868e4f59179:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T17:17:28,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:28,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86cf735dc0f15035813b7868e4f59179:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T17:17:28,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:17:28,357 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T17:17:28,357 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): 86cf735dc0f15035813b7868e4f59179/B is initiating minor compaction (all files) 2024-12-03T17:17:28,357 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86cf735dc0f15035813b7868e4f59179/B in TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:28,358 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/c320b220742c4ae7937322c33eda32a7, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/4cf4ff6088a245deadaf3f3df91b5c03, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/fac0875a75ed4c7a91cc6561917b2be8, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/954c951e23214cfe908a22934920f6bc] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp, totalSize=46.9 K 2024-12-03T17:17:28,358 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T17:17:28,358 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): 86cf735dc0f15035813b7868e4f59179/A is initiating minor compaction (all files) 2024-12-03T17:17:28,358 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86cf735dc0f15035813b7868e4f59179/A in TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:28,358 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/48e4f5cae8f848a9aa711765b4bd4184, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/80b8a8f2d55041d5af075f0e947512e1, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/a0bc32c6219f4f7eb594eeff575375af, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/04ec9678e612435886d5866904f507ff] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp, totalSize=137.7 K 2024-12-03T17:17:28,358 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:28,358 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. files: [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/48e4f5cae8f848a9aa711765b4bd4184, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/80b8a8f2d55041d5af075f0e947512e1, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/a0bc32c6219f4f7eb594eeff575375af, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/04ec9678e612435886d5866904f507ff] 2024-12-03T17:17:28,358 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting c320b220742c4ae7937322c33eda32a7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733246243375 2024-12-03T17:17:28,359 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48e4f5cae8f848a9aa711765b4bd4184, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733246243375 2024-12-03T17:17:28,359 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cf4ff6088a245deadaf3f3df91b5c03, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1733246243516 2024-12-03T17:17:28,359 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80b8a8f2d55041d5af075f0e947512e1, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1733246243516 2024-12-03T17:17:28,359 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting fac0875a75ed4c7a91cc6561917b2be8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733246244249 2024-12-03T17:17:28,359 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0bc32c6219f4f7eb594eeff575375af, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733246244249 2024-12-03T17:17:28,359 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 954c951e23214cfe908a22934920f6bc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1733246246582 2024-12-03T17:17:28,361 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04ec9678e612435886d5866904f507ff, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1733246246580 2024-12-03T17:17:28,385 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86cf735dc0f15035813b7868e4f59179#B#compaction#291 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:17:28,386 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/6eed8cd057614c768e550d8034cfd2ff is 50, key is test_row_0/B:col10/1733246247281/Put/seqid=0 2024-12-03T17:17:28,395 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=86cf735dc0f15035813b7868e4f59179] 2024-12-03T17:17:28,400 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203bdd636b939324d10b37f8cd656b99d59_86cf735dc0f15035813b7868e4f59179 store=[table=TestAcidGuarantees family=A region=86cf735dc0f15035813b7868e4f59179] 2024-12-03T17:17:28,403 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203bdd636b939324d10b37f8cd656b99d59_86cf735dc0f15035813b7868e4f59179, store=[table=TestAcidGuarantees family=A region=86cf735dc0f15035813b7868e4f59179] 2024-12-03T17:17:28,403 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203bdd636b939324d10b37f8cd656b99d59_86cf735dc0f15035813b7868e4f59179 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=86cf735dc0f15035813b7868e4f59179] 2024-12-03T17:17:28,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742164_1340 (size=12139) 2024-12-03T17:17:28,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742165_1341 (size=4469) 2024-12-03T17:17:28,468 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86cf735dc0f15035813b7868e4f59179#A#compaction#292 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:17:28,469 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/ed1678830d1e4900ad8bf316fbd70150 is 175, key is test_row_0/A:col10/1733246247281/Put/seqid=0 2024-12-03T17:17:28,474 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/6eed8cd057614c768e550d8034cfd2ff as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/6eed8cd057614c768e550d8034cfd2ff 2024-12-03T17:17:28,487 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 86cf735dc0f15035813b7868e4f59179/B of 86cf735dc0f15035813b7868e4f59179 into 6eed8cd057614c768e550d8034cfd2ff(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:17:28,487 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86cf735dc0f15035813b7868e4f59179: 2024-12-03T17:17:28,487 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., storeName=86cf735dc0f15035813b7868e4f59179/B, priority=12, startTime=1733246248356; duration=0sec 2024-12-03T17:17:28,487 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:17:28,487 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86cf735dc0f15035813b7868e4f59179:B 2024-12-03T17:17:28,487 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T17:17:28,491 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T17:17:28,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-03T17:17:28,491 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): 86cf735dc0f15035813b7868e4f59179/C is initiating minor compaction (all files) 2024-12-03T17:17:28,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:28,491 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2902): Flushing 86cf735dc0f15035813b7868e4f59179 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-12-03T17:17:28,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=A 2024-12-03T17:17:28,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:28,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=B 2024-12-03T17:17:28,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:28,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=C 2024-12-03T17:17:28,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:28,492 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86cf735dc0f15035813b7868e4f59179/C in TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:28,494 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/97ccc85f920d49a085a0acf4e0585131, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/eae44ee6df8744ee8e77cec1545cc69d, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/89c68435e1cf473492410d7aaf6dff79, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/08690ce989124676ad79c8de07ec0e41] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp, totalSize=46.9 K 2024-12-03T17:17:28,495 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 97ccc85f920d49a085a0acf4e0585131, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733246243375 2024-12-03T17:17:28,495 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting eae44ee6df8744ee8e77cec1545cc69d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1733246243516 2024-12-03T17:17:28,495 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 89c68435e1cf473492410d7aaf6dff79, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733246244249 2024-12-03T17:17:28,496 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 08690ce989124676ad79c8de07ec0e41, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1733246246582 2024-12-03T17:17:28,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742166_1342 (size=31093) 2024-12-03T17:17:28,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412039cd1eac19f764f6e8da2e91c7ced977c_86cf735dc0f15035813b7868e4f59179 is 50, key is test_row_0/A:col10/1733246247326/Put/seqid=0 2024-12-03T17:17:28,513 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/ed1678830d1e4900ad8bf316fbd70150 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/ed1678830d1e4900ad8bf316fbd70150 2024-12-03T17:17:28,519 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 86cf735dc0f15035813b7868e4f59179/A of 86cf735dc0f15035813b7868e4f59179 into ed1678830d1e4900ad8bf316fbd70150(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:17:28,519 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86cf735dc0f15035813b7868e4f59179: 2024-12-03T17:17:28,519 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., storeName=86cf735dc0f15035813b7868e4f59179/A, priority=12, startTime=1733246248355; duration=0sec 2024-12-03T17:17:28,519 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:28,519 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86cf735dc0f15035813b7868e4f59179:A 2024-12-03T17:17:28,522 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86cf735dc0f15035813b7868e4f59179#C#compaction#294 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:17:28,523 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/9ef60ee4b00f4ace9cde568066251671 is 50, key is test_row_0/C:col10/1733246247281/Put/seqid=0 2024-12-03T17:17:28,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742167_1343 (size=12154) 2024-12-03T17:17:28,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:17:28,538 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412039cd1eac19f764f6e8da2e91c7ced977c_86cf735dc0f15035813b7868e4f59179 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039cd1eac19f764f6e8da2e91c7ced977c_86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:28,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/a669714aaae046a8a5efddd96c9098df, store: [table=TestAcidGuarantees family=A region=86cf735dc0f15035813b7868e4f59179] 2024-12-03T17:17:28,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/a669714aaae046a8a5efddd96c9098df is 175, key is test_row_0/A:col10/1733246247326/Put/seqid=0 2024-12-03T17:17:28,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742168_1344 (size=12139) 2024-12-03T17:17:28,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742169_1345 (size=30955) 2024-12-03T17:17:28,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:28,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. as already flushing 2024-12-03T17:17:28,558 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=100, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/a669714aaae046a8a5efddd96c9098df 2024-12-03T17:17:28,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/82e117a1199d47b0b30568724b7ace4c is 50, key is test_row_0/B:col10/1733246247326/Put/seqid=0 2024-12-03T17:17:28,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742170_1346 (size=12001) 2024-12-03T17:17:28,603 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/82e117a1199d47b0b30568724b7ace4c 2024-12-03T17:17:28,614 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733246248580, max timestamp=1733246248610Num uniques 859; 2024-12-03T17:17:28,614 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T17:17:28,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733246248580, max timestamp=1733246248610Num uniques 859; 2024-12-03T17:17:28,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T17:17:28,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/59711e2f42ca4a499ec356e07545bd78 is 50, key is test_row_0/C:col10/1733246247326/Put/seqid=0 2024-12-03T17:17:28,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733246248580, max timestamp=1733246248610Num uniques 859; 2024-12-03T17:17:28,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T17:17:28,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742171_1347 (size=12001) 2024-12-03T17:17:28,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-03T17:17:28,755 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733246248719, max timestamp=1733246248748Num uniques 1118; 2024-12-03T17:17:28,755 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-12-03T17:17:28,755 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733246248719, max timestamp=1733246248748Num uniques 1118; 2024-12-03T17:17:28,755 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-12-03T17:17:28,759 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733246248719, max timestamp=1733246248748Num uniques 1118; 2024-12-03T17:17:28,759 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-12-03T17:17:28,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733246248826, max timestamp=1733246248872Num uniques 1027; 2024-12-03T17:17:28,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733246248826, max timestamp=1733246248872Num uniques 1027; 2024-12-03T17:17:28,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=12682, suffix heap size=134776,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=277 2024-12-03T17:17:28,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=12682, suffix heap size=134776,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=277 2024-12-03T17:17:28,881 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733246248826, max timestamp=1733246248872Num uniques 1027; 2024-12-03T17:17:28,881 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-03T17:17:28,955 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/9ef60ee4b00f4ace9cde568066251671 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/9ef60ee4b00f4ace9cde568066251671 2024-12-03T17:17:28,964 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 86cf735dc0f15035813b7868e4f59179/C of 86cf735dc0f15035813b7868e4f59179 into 9ef60ee4b00f4ace9cde568066251671(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:17:28,964 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86cf735dc0f15035813b7868e4f59179: 2024-12-03T17:17:28,964 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., storeName=86cf735dc0f15035813b7868e4f59179/C, priority=12, startTime=1733246248356; duration=0sec 2024-12-03T17:17:28,964 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:28,964 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86cf735dc0f15035813b7868e4f59179:C 2024-12-03T17:17:28,990 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733246248962, max timestamp=1733246248985Num uniques 1136; 2024-12-03T17:17:28,990 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=10806, suffix heap size=146112,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=236 2024-12-03T17:17:28,990 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733246248962, max timestamp=1733246248985Num uniques 1086; 2024-12-03T17:17:28,990 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-12-03T17:17:28,990 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733246248962, max timestamp=1733246248985Num uniques 1136; 2024-12-03T17:17:28,990 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=10806, suffix heap size=146112,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=236 2024-12-03T17:17:29,062 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/59711e2f42ca4a499ec356e07545bd78 2024-12-03T17:17:29,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/a669714aaae046a8a5efddd96c9098df as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/a669714aaae046a8a5efddd96c9098df 2024-12-03T17:17:29,087 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/a669714aaae046a8a5efddd96c9098df, entries=150, sequenceid=100, filesize=30.2 K 2024-12-03T17:17:29,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/82e117a1199d47b0b30568724b7ace4c as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/82e117a1199d47b0b30568724b7ace4c 2024-12-03T17:17:29,095 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733246249048, max timestamp=1733246249088Num uniques 1095; 2024-12-03T17:17:29,096 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-12-03T17:17:29,099 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733246249048, max timestamp=1733246249088Num uniques 1095; 2024-12-03T17:17:29,099 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-12-03T17:17:29,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733246249048, max timestamp=1733246249088Num uniques 1095; 2024-12-03T17:17:29,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-12-03T17:17:29,105 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/82e117a1199d47b0b30568724b7ace4c, entries=150, sequenceid=100, filesize=11.7 K 2024-12-03T17:17:29,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/59711e2f42ca4a499ec356e07545bd78 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/59711e2f42ca4a499ec356e07545bd78 2024-12-03T17:17:29,126 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/59711e2f42ca4a499ec356e07545bd78, entries=150, sequenceid=100, filesize=11.7 K 2024-12-03T17:17:29,129 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=80.51 KB/82440 for 86cf735dc0f15035813b7868e4f59179 in 638ms, sequenceid=100, compaction requested=false 2024-12-03T17:17:29,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for 86cf735dc0f15035813b7868e4f59179: 2024-12-03T17:17:29,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:29,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-12-03T17:17:29,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-12-03T17:17:29,132 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86cf735dc0f15035813b7868e4f59179 3/3 column families, dataSize=93.93 KB heapSize=257.39 KB 2024-12-03T17:17:29,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=A 2024-12-03T17:17:29,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:17:29,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=B 2024-12-03T17:17:29,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:17:29,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=C 2024-12-03T17:17:29,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:17:29,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:29,134 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-12-03T17:17:29,134 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5950 sec 2024-12-03T17:17:29,137 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 1.6020 sec 2024-12-03T17:17:29,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412030b5e5e26f6e3469887abc12f10e51d84_86cf735dc0f15035813b7868e4f59179 is 50, key is test_row_0/A:col10/1733246249125/Put/seqid=0 2024-12-03T17:17:29,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742172_1348 (size=22268) 2024-12-03T17:17:29,192 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:17:29,198 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412030b5e5e26f6e3469887abc12f10e51d84_86cf735dc0f15035813b7868e4f59179 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412030b5e5e26f6e3469887abc12f10e51d84_86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:29,200 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/0552189a8d044c18900cf3ab0dc43d23, store: [table=TestAcidGuarantees family=A region=86cf735dc0f15035813b7868e4f59179] 2024-12-03T17:17:29,200 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/0552189a8d044c18900cf3ab0dc43d23 is 175, key is test_row_0/A:col10/1733246249125/Put/seqid=0 2024-12-03T17:17:29,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742173_1349 (size=65673) 2024-12-03T17:17:29,232 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=200, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/0552189a8d044c18900cf3ab0dc43d23 2024-12-03T17:17:29,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/005d3848b3884d47957bbb6fb6950e7f is 50, key is test_row_0/B:col10/1733246249125/Put/seqid=0 2024-12-03T17:17:29,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742174_1350 (size=12151) 2024-12-03T17:17:29,261 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/005d3848b3884d47957bbb6fb6950e7f 2024-12-03T17:17:29,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:29,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246309235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:29,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:29,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246309237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:29,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:29,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:29,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:29,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:29,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:29,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:29,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:29,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246309266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:29,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:29,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:29,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:29,284 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/42a218214b09455c871bbc056c159ba7 is 50, key is test_row_0/C:col10/1733246249125/Put/seqid=0 2024-12-03T17:17:29,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742175_1351 (size=12151) 2024-12-03T17:17:29,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:29,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246309375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:29,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:29,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:29,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:29,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:29,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246309378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:29,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:29,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:29,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:29,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:29,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246309394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:29,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:29,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:29,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:29,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:29,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246309594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:29,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:29,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:29,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:29,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:29,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246309608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:29,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:29,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:29,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:29,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:29,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246309614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:29,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:29,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:29,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:29,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-03T17:17:29,695 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:17:29,696 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T17:17:29,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-12-03T17:17:29,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-03T17:17:29,698 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T17:17:29,699 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T17:17:29,699 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T17:17:29,710 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/42a218214b09455c871bbc056c159ba7 2024-12-03T17:17:29,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/0552189a8d044c18900cf3ab0dc43d23 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/0552189a8d044c18900cf3ab0dc43d23 2024-12-03T17:17:29,768 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/0552189a8d044c18900cf3ab0dc43d23, entries=350, sequenceid=200, filesize=64.1 K 2024-12-03T17:17:29,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/005d3848b3884d47957bbb6fb6950e7f as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/005d3848b3884d47957bbb6fb6950e7f 2024-12-03T17:17:29,780 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/005d3848b3884d47957bbb6fb6950e7f, entries=150, sequenceid=200, filesize=11.9 K 2024-12-03T17:17:29,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/42a218214b09455c871bbc056c159ba7 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/42a218214b09455c871bbc056c159ba7 2024-12-03T17:17:29,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-03T17:17:29,823 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/42a218214b09455c871bbc056c159ba7, entries=150, sequenceid=200, filesize=11.9 K 2024-12-03T17:17:29,824 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~257.34 KB/263520, currentSize=87.22 KB/89310 for 86cf735dc0f15035813b7868e4f59179 in 692ms, sequenceid=200, compaction requested=true 2024-12-03T17:17:29,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86cf735dc0f15035813b7868e4f59179: 2024-12-03T17:17:29,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86cf735dc0f15035813b7868e4f59179:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T17:17:29,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:29,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86cf735dc0f15035813b7868e4f59179:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T17:17:29,825 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:17:29,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:17:29,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86cf735dc0f15035813b7868e4f59179:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T17:17:29,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T17:17:29,826 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:17:29,829 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:17:29,830 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): 86cf735dc0f15035813b7868e4f59179/A is initiating minor compaction (all files) 2024-12-03T17:17:29,830 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86cf735dc0f15035813b7868e4f59179/A in TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:29,830 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/ed1678830d1e4900ad8bf316fbd70150, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/a669714aaae046a8a5efddd96c9098df, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/0552189a8d044c18900cf3ab0dc43d23] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp, totalSize=124.7 K 2024-12-03T17:17:29,830 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:29,830 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. files: [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/ed1678830d1e4900ad8bf316fbd70150, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/a669714aaae046a8a5efddd96c9098df, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/0552189a8d044c18900cf3ab0dc43d23] 2024-12-03T17:17:29,833 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed1678830d1e4900ad8bf316fbd70150, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1733246246582 2024-12-03T17:17:29,834 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:17:29,834 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): 86cf735dc0f15035813b7868e4f59179/B is initiating minor compaction (all files) 2024-12-03T17:17:29,834 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86cf735dc0f15035813b7868e4f59179/B in TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:29,835 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/6eed8cd057614c768e550d8034cfd2ff, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/82e117a1199d47b0b30568724b7ace4c, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/005d3848b3884d47957bbb6fb6950e7f] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp, totalSize=35.4 K 2024-12-03T17:17:29,835 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 6eed8cd057614c768e550d8034cfd2ff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1733246246582 2024-12-03T17:17:29,837 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting a669714aaae046a8a5efddd96c9098df, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1733246247299 2024-12-03T17:17:29,839 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 82e117a1199d47b0b30568724b7ace4c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1733246247299 2024-12-03T17:17:29,840 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 005d3848b3884d47957bbb6fb6950e7f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733246249124 2024-12-03T17:17:29,846 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0552189a8d044c18900cf3ab0dc43d23, keycount=350, bloomtype=ROW, size=64.1 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733246249059 2024-12-03T17:17:29,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-03T17:17:29,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:29,858 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing 86cf735dc0f15035813b7868e4f59179 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-03T17:17:29,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=A 2024-12-03T17:17:29,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:29,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=B 2024-12-03T17:17:29,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:29,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=C 2024-12-03T17:17:29,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:29,902 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=86cf735dc0f15035813b7868e4f59179] 2024-12-03T17:17:29,902 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86cf735dc0f15035813b7868e4f59179#B#compaction#300 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:17:29,903 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/54191cdcf9134e2d92e78800ba0903b0 is 50, key is test_row_0/B:col10/1733246249125/Put/seqid=0 2024-12-03T17:17:29,947 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203edb32b34dad74f2ca62892b810847318_86cf735dc0f15035813b7868e4f59179 store=[table=TestAcidGuarantees family=A region=86cf735dc0f15035813b7868e4f59179] 2024-12-03T17:17:29,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. as already flushing 2024-12-03T17:17:29,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:29,949 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203edb32b34dad74f2ca62892b810847318_86cf735dc0f15035813b7868e4f59179, store=[table=TestAcidGuarantees family=A region=86cf735dc0f15035813b7868e4f59179] 2024-12-03T17:17:29,949 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203edb32b34dad74f2ca62892b810847318_86cf735dc0f15035813b7868e4f59179 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=86cf735dc0f15035813b7868e4f59179] 2024-12-03T17:17:29,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742176_1352 (size=12391) 2024-12-03T17:17:29,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412039aba100513c64982aa5e0a49dfec354f_86cf735dc0f15035813b7868e4f59179 is 50, key is test_row_0/A:col10/1733246249198/Put/seqid=0 2024-12-03T17:17:30,011 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/54191cdcf9134e2d92e78800ba0903b0 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/54191cdcf9134e2d92e78800ba0903b0 2024-12-03T17:17:30,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-03T17:17:30,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742177_1353 (size=4469) 2024-12-03T17:17:30,021 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86cf735dc0f15035813b7868e4f59179/B of 86cf735dc0f15035813b7868e4f59179 into 54191cdcf9134e2d92e78800ba0903b0(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:17:30,021 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86cf735dc0f15035813b7868e4f59179: 2024-12-03T17:17:30,022 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., storeName=86cf735dc0f15035813b7868e4f59179/B, priority=13, startTime=1733246249825; duration=0sec 2024-12-03T17:17:30,022 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:17:30,022 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86cf735dc0f15035813b7868e4f59179:B 2024-12-03T17:17:30,022 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:17:30,022 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86cf735dc0f15035813b7868e4f59179#A#compaction#301 average throughput is 0.20 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:17:30,023 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/07ca3656577e4ede8607f9502946da14 is 175, key is test_row_0/A:col10/1733246249125/Put/seqid=0 2024-12-03T17:17:30,027 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:17:30,027 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): 86cf735dc0f15035813b7868e4f59179/C is initiating minor compaction (all files) 2024-12-03T17:17:30,027 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86cf735dc0f15035813b7868e4f59179/C in TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:30,027 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/9ef60ee4b00f4ace9cde568066251671, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/59711e2f42ca4a499ec356e07545bd78, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/42a218214b09455c871bbc056c159ba7] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp, totalSize=35.4 K 2024-12-03T17:17:30,028 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ef60ee4b00f4ace9cde568066251671, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1733246246582 2024-12-03T17:17:30,029 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 59711e2f42ca4a499ec356e07545bd78, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1733246247299 2024-12-03T17:17:30,029 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 42a218214b09455c871bbc056c159ba7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733246249124 2024-12-03T17:17:30,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742178_1354 (size=12304) 2024-12-03T17:17:30,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:17:30,054 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412039aba100513c64982aa5e0a49dfec354f_86cf735dc0f15035813b7868e4f59179 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039aba100513c64982aa5e0a49dfec354f_86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:30,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/21f739896e614231b7d90b92b9762104, store: [table=TestAcidGuarantees family=A region=86cf735dc0f15035813b7868e4f59179] 2024-12-03T17:17:30,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/21f739896e614231b7d90b92b9762104 is 175, key is test_row_0/A:col10/1733246249198/Put/seqid=0 2024-12-03T17:17:30,078 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86cf735dc0f15035813b7868e4f59179#C#compaction#303 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:17:30,079 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/b90a5cd0e6c94fbfbbc093f711e272cf is 50, key is test_row_0/C:col10/1733246249125/Put/seqid=0 2024-12-03T17:17:30,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:30,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246310044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:30,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:30,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:30,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:30,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:30,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246310083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:30,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:30,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:30,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:30,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:30,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246310087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:30,122 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:30,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:30,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:30,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742180_1356 (size=31105) 2024-12-03T17:17:30,134 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=216, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/21f739896e614231b7d90b92b9762104 2024-12-03T17:17:30,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742181_1357 (size=12391) 2024-12-03T17:17:30,146 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/b90a5cd0e6c94fbfbbc093f711e272cf as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/b90a5cd0e6c94fbfbbc093f711e272cf 2024-12-03T17:17:30,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742179_1355 (size=31345) 2024-12-03T17:17:30,154 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86cf735dc0f15035813b7868e4f59179/C of 86cf735dc0f15035813b7868e4f59179 into b90a5cd0e6c94fbfbbc093f711e272cf(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:17:30,154 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86cf735dc0f15035813b7868e4f59179: 2024-12-03T17:17:30,154 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., storeName=86cf735dc0f15035813b7868e4f59179/C, priority=13, startTime=1733246249825; duration=0sec 2024-12-03T17:17:30,154 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:30,154 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86cf735dc0f15035813b7868e4f59179:C 2024-12-03T17:17:30,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/49704ba960854de381fabd58c030851e is 50, key is test_row_0/B:col10/1733246249198/Put/seqid=0 2024-12-03T17:17:30,161 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/07ca3656577e4ede8607f9502946da14 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/07ca3656577e4ede8607f9502946da14 2024-12-03T17:17:30,167 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86cf735dc0f15035813b7868e4f59179/A of 86cf735dc0f15035813b7868e4f59179 into 07ca3656577e4ede8607f9502946da14(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:17:30,167 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86cf735dc0f15035813b7868e4f59179: 2024-12-03T17:17:30,167 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., storeName=86cf735dc0f15035813b7868e4f59179/A, priority=13, startTime=1733246249824; duration=0sec 2024-12-03T17:17:30,167 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:30,167 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86cf735dc0f15035813b7868e4f59179:A 2024-12-03T17:17:30,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742182_1358 (size=12151) 2024-12-03T17:17:30,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:30,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246310196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:30,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:30,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:30,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:30,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:30,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246310227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:30,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:30,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:30,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:30,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:30,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246310235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:30,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:30,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:30,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:30,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-03T17:17:30,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:30,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246310423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:30,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:30,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:30,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:30,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:30,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246310443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:30,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:30,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:30,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:30,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:30,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246310463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:30,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:30,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:30,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:30,592 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/49704ba960854de381fabd58c030851e 2024-12-03T17:17:30,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/29fe3c384e67415298a897fc6401babe is 50, key is test_row_0/C:col10/1733246249198/Put/seqid=0 2024-12-03T17:17:30,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742183_1359 (size=12151) 2024-12-03T17:17:30,639 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/29fe3c384e67415298a897fc6401babe 2024-12-03T17:17:30,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/21f739896e614231b7d90b92b9762104 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/21f739896e614231b7d90b92b9762104 2024-12-03T17:17:30,660 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/21f739896e614231b7d90b92b9762104, entries=150, sequenceid=216, filesize=30.4 K 2024-12-03T17:17:30,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/49704ba960854de381fabd58c030851e as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/49704ba960854de381fabd58c030851e 2024-12-03T17:17:30,669 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/49704ba960854de381fabd58c030851e, entries=150, sequenceid=216, filesize=11.9 K 2024-12-03T17:17:30,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/29fe3c384e67415298a897fc6401babe as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/29fe3c384e67415298a897fc6401babe 2024-12-03T17:17:30,675 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/29fe3c384e67415298a897fc6401babe, entries=150, sequenceid=216, filesize=11.9 K 2024-12-03T17:17:30,676 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 86cf735dc0f15035813b7868e4f59179 in 818ms, sequenceid=216, compaction requested=false 2024-12-03T17:17:30,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for 86cf735dc0f15035813b7868e4f59179: 2024-12-03T17:17:30,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:30,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-12-03T17:17:30,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-12-03T17:17:30,680 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-12-03T17:17:30,680 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 979 msec 2024-12-03T17:17:30,682 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 985 msec 2024-12-03T17:17:30,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:30,745 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86cf735dc0f15035813b7868e4f59179 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-03T17:17:30,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=A 2024-12-03T17:17:30,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:30,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=B 2024-12-03T17:17:30,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:30,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=C 2024-12-03T17:17:30,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:30,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203eab4be1447fc413daa9be2e4bbd37252_86cf735dc0f15035813b7868e4f59179 is 50, key is test_row_0/A:col10/1733246250036/Put/seqid=0 2024-12-03T17:17:30,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742184_1360 (size=14794) 2024-12-03T17:17:30,814 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:17:30,827 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203eab4be1447fc413daa9be2e4bbd37252_86cf735dc0f15035813b7868e4f59179 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203eab4be1447fc413daa9be2e4bbd37252_86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:30,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-03T17:17:30,834 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/fde5c62620934e9886ef6c08f59369d9, store: [table=TestAcidGuarantees family=A region=86cf735dc0f15035813b7868e4f59179] 2024-12-03T17:17:30,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/fde5c62620934e9886ef6c08f59369d9 is 175, key is test_row_0/A:col10/1733246250036/Put/seqid=0 2024-12-03T17:17:30,836 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:17:30,839 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T17:17:30,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-12-03T17:17:30,844 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T17:17:30,845 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T17:17:30,845 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T17:17:30,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-03T17:17:30,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742185_1361 (size=39749) 2024-12-03T17:17:30,883 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=236, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/fde5c62620934e9886ef6c08f59369d9 2024-12-03T17:17:30,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:30,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246310877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:30,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:30,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:30,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:30,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:30,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246310886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:30,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:30,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246310891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:30,908 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:30,908 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:30,908 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:30,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/594a9894afe34f77965b9d859b79d52a is 50, key is test_row_0/B:col10/1733246250036/Put/seqid=0 2024-12-03T17:17:30,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:30,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:30,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:30,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742186_1362 (size=12151) 2024-12-03T17:17:30,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-03T17:17:31,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-03T17:17:31,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:31,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. as already flushing 2024-12-03T17:17:31,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:31,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:31,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:31,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:31,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:31,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246311014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:31,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:31,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:31,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246311015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:31,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:31,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:31,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:31,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:31,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:31,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:31,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246311015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:31,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:31,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:31,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:31,125 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-03T17:17:31,125 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-03T17:17:31,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-03T17:17:31,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-03T17:17:31,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:31,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. as already flushing 2024-12-03T17:17:31,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:31,169 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:31,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:31,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:31,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:31,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246311223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:31,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:31,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:31,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246311223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:31,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:31,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:31,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:31,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:31,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:31,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:31,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246311235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:31,245 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:31,245 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:31,245 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:31,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-03T17:17:31,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:31,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. as already flushing 2024-12-03T17:17:31,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:31,334 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:31,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:31,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:31,349 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/594a9894afe34f77965b9d859b79d52a 2024-12-03T17:17:31,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/483026c942b04cb0b760489a4c51ab7f is 50, key is test_row_0/C:col10/1733246250036/Put/seqid=0 2024-12-03T17:17:31,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742187_1363 (size=12151) 2024-12-03T17:17:31,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-03T17:17:31,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-03T17:17:31,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:31,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. as already flushing 2024-12-03T17:17:31,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:31,492 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:31,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:31,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:31,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:31,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246311543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:31,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:31,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:31,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:31,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:31,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246311545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:31,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:31,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:31,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:31,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:31,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246311554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:31,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:31,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:31,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:31,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-03T17:17:31,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:31,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. as already flushing 2024-12-03T17:17:31,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:31,650 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:31,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:31,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:31,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:31,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51304 deadline: 1733246311734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:31,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:31,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51294 deadline: 1733246311744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:31,745 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 27e0f852a164,41167,1733246151593 for 'test_row_2' in 86cf735dc0f15035813b7868e4f59179 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8318 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T17:17:31,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:31,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:31,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:31,746 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 27e0f852a164,41167,1733246151593 for 'test_row_2' in 86cf735dc0f15035813b7868e4f59179 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8310 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T17:17:31,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:31,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:31,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:31,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-03T17:17:31,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:31,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. as already flushing 2024-12-03T17:17:31,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:31,814 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:31,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:31,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:31,818 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/483026c942b04cb0b760489a4c51ab7f 2024-12-03T17:17:31,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/fde5c62620934e9886ef6c08f59369d9 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/fde5c62620934e9886ef6c08f59369d9 2024-12-03T17:17:31,830 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/fde5c62620934e9886ef6c08f59369d9, entries=200, sequenceid=236, filesize=38.8 K 2024-12-03T17:17:31,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/594a9894afe34f77965b9d859b79d52a as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/594a9894afe34f77965b9d859b79d52a 2024-12-03T17:17:31,835 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/594a9894afe34f77965b9d859b79d52a, entries=150, sequenceid=236, filesize=11.9 K 2024-12-03T17:17:31,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/483026c942b04cb0b760489a4c51ab7f as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/483026c942b04cb0b760489a4c51ab7f 2024-12-03T17:17:31,839 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/483026c942b04cb0b760489a4c51ab7f, entries=150, sequenceid=236, filesize=11.9 K 2024-12-03T17:17:31,840 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=93.93 KB/96180 for 86cf735dc0f15035813b7868e4f59179 in 1095ms, sequenceid=236, compaction requested=true 2024-12-03T17:17:31,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86cf735dc0f15035813b7868e4f59179: 2024-12-03T17:17:31,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86cf735dc0f15035813b7868e4f59179:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T17:17:31,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:31,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86cf735dc0f15035813b7868e4f59179:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T17:17:31,840 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:17:31,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:31,840 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:17:31,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86cf735dc0f15035813b7868e4f59179:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T17:17:31,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:17:31,843 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102199 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:17:31,843 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:17:31,843 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): 86cf735dc0f15035813b7868e4f59179/B is initiating minor compaction (all files) 2024-12-03T17:17:31,843 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): 86cf735dc0f15035813b7868e4f59179/A is initiating minor compaction (all files) 2024-12-03T17:17:31,844 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86cf735dc0f15035813b7868e4f59179/B in TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:31,844 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86cf735dc0f15035813b7868e4f59179/A in TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:31,844 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/07ca3656577e4ede8607f9502946da14, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/21f739896e614231b7d90b92b9762104, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/fde5c62620934e9886ef6c08f59369d9] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp, totalSize=99.8 K 2024-12-03T17:17:31,844 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/54191cdcf9134e2d92e78800ba0903b0, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/49704ba960854de381fabd58c030851e, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/594a9894afe34f77965b9d859b79d52a] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp, totalSize=35.8 K 2024-12-03T17:17:31,844 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:31,844 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. files: [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/07ca3656577e4ede8607f9502946da14, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/21f739896e614231b7d90b92b9762104, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/fde5c62620934e9886ef6c08f59369d9] 2024-12-03T17:17:31,844 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 54191cdcf9134e2d92e78800ba0903b0, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733246249124 2024-12-03T17:17:31,844 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07ca3656577e4ede8607f9502946da14, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733246249124 2024-12-03T17:17:31,846 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21f739896e614231b7d90b92b9762104, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1733246249198 2024-12-03T17:17:31,846 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 49704ba960854de381fabd58c030851e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1733246249198 2024-12-03T17:17:31,846 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 594a9894afe34f77965b9d859b79d52a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1733246250036 2024-12-03T17:17:31,846 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting fde5c62620934e9886ef6c08f59369d9, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1733246250036 2024-12-03T17:17:31,851 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=86cf735dc0f15035813b7868e4f59179] 2024-12-03T17:17:31,852 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86cf735dc0f15035813b7868e4f59179#B#compaction#309 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:17:31,852 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203bfba627f77da4434acc894a0105e9267_86cf735dc0f15035813b7868e4f59179 store=[table=TestAcidGuarantees family=A region=86cf735dc0f15035813b7868e4f59179] 2024-12-03T17:17:31,853 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/c570211cfe0b4e3896d54bd981ca7fba is 50, key is test_row_0/B:col10/1733246250036/Put/seqid=0 2024-12-03T17:17:31,854 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203bfba627f77da4434acc894a0105e9267_86cf735dc0f15035813b7868e4f59179, store=[table=TestAcidGuarantees family=A region=86cf735dc0f15035813b7868e4f59179] 2024-12-03T17:17:31,854 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203bfba627f77da4434acc894a0105e9267_86cf735dc0f15035813b7868e4f59179 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=86cf735dc0f15035813b7868e4f59179] 2024-12-03T17:17:31,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742188_1364 (size=12493) 2024-12-03T17:17:31,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742189_1365 (size=4469) 2024-12-03T17:17:31,861 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86cf735dc0f15035813b7868e4f59179#A#compaction#310 average throughput is 2.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:17:31,862 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/81ba4302f14f487891f64fee38f3a05d is 175, key is test_row_0/A:col10/1733246250036/Put/seqid=0 2024-12-03T17:17:31,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742190_1366 (size=31447) 2024-12-03T17:17:31,887 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/81ba4302f14f487891f64fee38f3a05d as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/81ba4302f14f487891f64fee38f3a05d 2024-12-03T17:17:31,898 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86cf735dc0f15035813b7868e4f59179/A of 86cf735dc0f15035813b7868e4f59179 into 81ba4302f14f487891f64fee38f3a05d(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:17:31,898 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86cf735dc0f15035813b7868e4f59179: 2024-12-03T17:17:31,898 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., storeName=86cf735dc0f15035813b7868e4f59179/A, priority=13, startTime=1733246251840; duration=0sec 2024-12-03T17:17:31,898 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:17:31,898 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86cf735dc0f15035813b7868e4f59179:A 2024-12-03T17:17:31,898 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:17:31,899 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:17:31,899 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): 86cf735dc0f15035813b7868e4f59179/C is initiating minor compaction (all files) 2024-12-03T17:17:31,899 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86cf735dc0f15035813b7868e4f59179/C in TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:31,900 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/b90a5cd0e6c94fbfbbc093f711e272cf, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/29fe3c384e67415298a897fc6401babe, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/483026c942b04cb0b760489a4c51ab7f] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp, totalSize=35.8 K 2024-12-03T17:17:31,900 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting b90a5cd0e6c94fbfbbc093f711e272cf, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733246249124 2024-12-03T17:17:31,900 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29fe3c384e67415298a897fc6401babe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1733246249198 2024-12-03T17:17:31,901 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 483026c942b04cb0b760489a4c51ab7f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1733246250036 2024-12-03T17:17:31,913 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86cf735dc0f15035813b7868e4f59179#C#compaction#311 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:17:31,914 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/151107f4668e471586a9f345e5ea4d97 is 50, key is test_row_0/C:col10/1733246250036/Put/seqid=0 2024-12-03T17:17:31,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742191_1367 (size=12493) 2024-12-03T17:17:31,963 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/151107f4668e471586a9f345e5ea4d97 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/151107f4668e471586a9f345e5ea4d97 2024-12-03T17:17:31,969 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86cf735dc0f15035813b7868e4f59179/C of 86cf735dc0f15035813b7868e4f59179 into 151107f4668e471586a9f345e5ea4d97(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:17:31,969 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86cf735dc0f15035813b7868e4f59179: 2024-12-03T17:17:31,969 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., storeName=86cf735dc0f15035813b7868e4f59179/C, priority=13, startTime=1733246251840; duration=0sec 2024-12-03T17:17:31,969 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:31,969 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86cf735dc0f15035813b7868e4f59179:C 2024-12-03T17:17:31,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-03T17:17:31,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:31,970 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 86cf735dc0f15035813b7868e4f59179 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-03T17:17:31,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=A 2024-12-03T17:17:31,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:31,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=B 2024-12-03T17:17:31,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:31,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=C 2024-12-03T17:17:31,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:31,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203fd7a03d3b0cf4cb589f0ab5dab1b36c3_86cf735dc0f15035813b7868e4f59179 is 50, key is test_row_0/A:col10/1733246250847/Put/seqid=0 2024-12-03T17:17:31,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-03T17:17:32,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742192_1368 (size=12304) 2024-12-03T17:17:32,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:17:32,036 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203fd7a03d3b0cf4cb589f0ab5dab1b36c3_86cf735dc0f15035813b7868e4f59179 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203fd7a03d3b0cf4cb589f0ab5dab1b36c3_86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:32,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/701ebf0ef67a43eebe8536582d86c300, store: [table=TestAcidGuarantees family=A region=86cf735dc0f15035813b7868e4f59179] 2024-12-03T17:17:32,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/701ebf0ef67a43eebe8536582d86c300 is 175, key is test_row_0/A:col10/1733246250847/Put/seqid=0 2024-12-03T17:17:32,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742193_1369 (size=31105) 2024-12-03T17:17:32,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:32,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. as already flushing 2024-12-03T17:17:32,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:32,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246312110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:32,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:32,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:32,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:32,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:32,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246312111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:32,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:32,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:32,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:32,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:32,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246312114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:32,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:32,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:32,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:32,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:32,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246312224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:32,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:32,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:32,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:32,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:32,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246312243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:32,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:32,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:32,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:32,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:32,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246312246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:32,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:32,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:32,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:32,294 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/c570211cfe0b4e3896d54bd981ca7fba as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/c570211cfe0b4e3896d54bd981ca7fba 2024-12-03T17:17:32,302 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86cf735dc0f15035813b7868e4f59179/B of 86cf735dc0f15035813b7868e4f59179 into c570211cfe0b4e3896d54bd981ca7fba(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:17:32,302 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86cf735dc0f15035813b7868e4f59179: 2024-12-03T17:17:32,302 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., storeName=86cf735dc0f15035813b7868e4f59179/B, priority=13, startTime=1733246251840; duration=0sec 2024-12-03T17:17:32,302 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:32,302 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86cf735dc0f15035813b7868e4f59179:B 2024-12-03T17:17:32,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:32,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246312434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:32,482 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=255, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/701ebf0ef67a43eebe8536582d86c300 2024-12-03T17:17:32,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:32,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:32,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:32,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:32,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246312481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:32,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:32,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246312481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:32,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:32,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:32,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:32,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:32,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:32,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:32,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/44ce70fb0c774073bef4639afed0465e is 50, key is test_row_0/B:col10/1733246250847/Put/seqid=0 2024-12-03T17:17:32,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742194_1370 (size=12151) 2024-12-03T17:17:32,510 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/44ce70fb0c774073bef4639afed0465e 2024-12-03T17:17:32,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/fcf9a936139a43b8bc7f6a8932a9ccf2 is 50, key is test_row_0/C:col10/1733246250847/Put/seqid=0 2024-12-03T17:17:32,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742195_1371 (size=12151) 2024-12-03T17:17:32,588 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/fcf9a936139a43b8bc7f6a8932a9ccf2 2024-12-03T17:17:32,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/701ebf0ef67a43eebe8536582d86c300 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/701ebf0ef67a43eebe8536582d86c300 2024-12-03T17:17:32,625 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/701ebf0ef67a43eebe8536582d86c300, entries=150, sequenceid=255, filesize=30.4 K 2024-12-03T17:17:32,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/44ce70fb0c774073bef4639afed0465e as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/44ce70fb0c774073bef4639afed0465e 2024-12-03T17:17:32,641 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/44ce70fb0c774073bef4639afed0465e, entries=150, sequenceid=255, filesize=11.9 K 2024-12-03T17:17:32,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/fcf9a936139a43b8bc7f6a8932a9ccf2 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/fcf9a936139a43b8bc7f6a8932a9ccf2 2024-12-03T17:17:32,647 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/fcf9a936139a43b8bc7f6a8932a9ccf2, entries=150, sequenceid=255, filesize=11.9 K 2024-12-03T17:17:32,649 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 86cf735dc0f15035813b7868e4f59179 in 679ms, sequenceid=255, compaction requested=false 2024-12-03T17:17:32,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 86cf735dc0f15035813b7868e4f59179: 2024-12-03T17:17:32,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:32,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-12-03T17:17:32,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-12-03T17:17:32,653 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-12-03T17:17:32,653 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8060 sec 2024-12-03T17:17:32,658 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.8160 sec 2024-12-03T17:17:32,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:32,798 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86cf735dc0f15035813b7868e4f59179 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-03T17:17:32,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=A 2024-12-03T17:17:32,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:32,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=B 2024-12-03T17:17:32,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:32,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=C 2024-12-03T17:17:32,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:32,818 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412039aa066998fd040ab9203ce591d0cdba8_86cf735dc0f15035813b7868e4f59179 is 50, key is test_row_0/A:col10/1733246252100/Put/seqid=0 2024-12-03T17:17:32,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742196_1372 (size=14994) 2024-12-03T17:17:32,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:32,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246312857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:32,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:32,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:32,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:32,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:32,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246312871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:32,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:32,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:32,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:32,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:32,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246312874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:32,896 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:32,896 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:32,896 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:32,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:32,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246312986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:32,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:32,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:32,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:32,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-03T17:17:32,994 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:17:32,994 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T17:17:33,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-12-03T17:17:33,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:33,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246312993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:33,005 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T17:17:33,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:33,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:33,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:33,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-03T17:17:33,009 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T17:17:33,009 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T17:17:33,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:33,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246313005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:33,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:33,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:33,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:33,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-03T17:17:33,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-03T17:17:33,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:33,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. as already flushing 2024-12-03T17:17:33,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:33,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:33,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:33,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:33,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:33,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246313194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:33,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:33,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:33,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:33,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:33,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246313219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:33,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:33,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:33,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:33,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:33,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246313234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:33,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:33,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:33,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:33,250 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:17:33,256 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412039aa066998fd040ab9203ce591d0cdba8_86cf735dc0f15035813b7868e4f59179 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039aa066998fd040ab9203ce591d0cdba8_86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:33,259 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/b168b6771648432ead8f092758847443, store: [table=TestAcidGuarantees family=A region=86cf735dc0f15035813b7868e4f59179] 2024-12-03T17:17:33,260 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/b168b6771648432ead8f092758847443 is 175, key is test_row_0/A:col10/1733246252100/Put/seqid=0 2024-12-03T17:17:33,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742197_1373 (size=39949) 2024-12-03T17:17:33,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-03T17:17:33,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-03T17:17:33,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:33,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. as already flushing 2024-12-03T17:17:33,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:33,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:33,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:33,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:33,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-03T17:17:33,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:33,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. as already flushing 2024-12-03T17:17:33,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:33,496 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:33,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:33,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:33,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:33,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246313523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:33,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:33,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:33,531 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:33,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:33,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246313544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:33,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:33,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:33,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:33,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:33,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246313555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:33,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:33,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:33,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:33,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-03T17:17:33,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-03T17:17:33,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:33,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. as already flushing 2024-12-03T17:17:33,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:33,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:33,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:33,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:33,701 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=272, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/b168b6771648432ead8f092758847443 2024-12-03T17:17:33,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/ae77c1dc366d47cd81677d176f9d20f0 is 50, key is test_row_0/B:col10/1733246252100/Put/seqid=0 2024-12-03T17:17:33,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742198_1374 (size=12301) 2024-12-03T17:17:33,773 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/ae77c1dc366d47cd81677d176f9d20f0 2024-12-03T17:17:33,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-03T17:17:33,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:33,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. as already flushing 2024-12-03T17:17:33,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:33,821 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:33,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:33,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:33,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/30873cc8521a48e78da1f70c50dcbb6f is 50, key is test_row_0/C:col10/1733246252100/Put/seqid=0 2024-12-03T17:17:33,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742199_1375 (size=12301) 2024-12-03T17:17:33,888 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/30873cc8521a48e78da1f70c50dcbb6f 2024-12-03T17:17:33,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/b168b6771648432ead8f092758847443 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/b168b6771648432ead8f092758847443 2024-12-03T17:17:33,902 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/b168b6771648432ead8f092758847443, entries=200, sequenceid=272, filesize=39.0 K 2024-12-03T17:17:33,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/ae77c1dc366d47cd81677d176f9d20f0 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/ae77c1dc366d47cd81677d176f9d20f0 2024-12-03T17:17:33,907 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/ae77c1dc366d47cd81677d176f9d20f0, entries=150, sequenceid=272, filesize=12.0 K 2024-12-03T17:17:33,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/30873cc8521a48e78da1f70c50dcbb6f as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/30873cc8521a48e78da1f70c50dcbb6f 2024-12-03T17:17:33,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/30873cc8521a48e78da1f70c50dcbb6f, entries=150, sequenceid=272, filesize=12.0 K 2024-12-03T17:17:33,923 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 86cf735dc0f15035813b7868e4f59179 in 1125ms, sequenceid=272, compaction requested=true 2024-12-03T17:17:33,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86cf735dc0f15035813b7868e4f59179: 2024-12-03T17:17:33,923 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:17:33,927 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102501 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:17:33,927 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): 86cf735dc0f15035813b7868e4f59179/A is initiating minor compaction (all files) 2024-12-03T17:17:33,927 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86cf735dc0f15035813b7868e4f59179/A in TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:33,928 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/81ba4302f14f487891f64fee38f3a05d, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/701ebf0ef67a43eebe8536582d86c300, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/b168b6771648432ead8f092758847443] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp, totalSize=100.1 K 2024-12-03T17:17:33,928 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:33,928 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. files: [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/81ba4302f14f487891f64fee38f3a05d, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/701ebf0ef67a43eebe8536582d86c300, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/b168b6771648432ead8f092758847443] 2024-12-03T17:17:33,929 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81ba4302f14f487891f64fee38f3a05d, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1733246250036 2024-12-03T17:17:33,930 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 701ebf0ef67a43eebe8536582d86c300, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733246250778 2024-12-03T17:17:33,930 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting b168b6771648432ead8f092758847443, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733246252100 2024-12-03T17:17:33,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86cf735dc0f15035813b7868e4f59179:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T17:17:33,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:33,946 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:17:33,947 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=86cf735dc0f15035813b7868e4f59179] 2024-12-03T17:17:33,948 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36945 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:17:33,948 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): 86cf735dc0f15035813b7868e4f59179/B is initiating minor compaction (all files) 2024-12-03T17:17:33,948 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86cf735dc0f15035813b7868e4f59179/B in TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:33,948 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/c570211cfe0b4e3896d54bd981ca7fba, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/44ce70fb0c774073bef4639afed0465e, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/ae77c1dc366d47cd81677d176f9d20f0] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp, totalSize=36.1 K 2024-12-03T17:17:33,949 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting c570211cfe0b4e3896d54bd981ca7fba, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1733246250036 2024-12-03T17:17:33,949 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 44ce70fb0c774073bef4639afed0465e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733246250778 2024-12-03T17:17:33,949 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting ae77c1dc366d47cd81677d176f9d20f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733246252100 2024-12-03T17:17:33,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86cf735dc0f15035813b7868e4f59179:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T17:17:33,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:33,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86cf735dc0f15035813b7868e4f59179:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T17:17:33,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:17:33,963 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120313d645a5ba274680a91bbbdaa238bca8_86cf735dc0f15035813b7868e4f59179 store=[table=TestAcidGuarantees family=A region=86cf735dc0f15035813b7868e4f59179] 2024-12-03T17:17:33,965 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120313d645a5ba274680a91bbbdaa238bca8_86cf735dc0f15035813b7868e4f59179, store=[table=TestAcidGuarantees family=A region=86cf735dc0f15035813b7868e4f59179] 2024-12-03T17:17:33,965 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120313d645a5ba274680a91bbbdaa238bca8_86cf735dc0f15035813b7868e4f59179 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=86cf735dc0f15035813b7868e4f59179] 2024-12-03T17:17:33,967 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86cf735dc0f15035813b7868e4f59179#B#compaction#319 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:17:33,968 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/9043b1271b744d1ab8343d4bb85b478e is 50, key is test_row_0/B:col10/1733246252100/Put/seqid=0 2024-12-03T17:17:33,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-03T17:17:33,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:33,979 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 86cf735dc0f15035813b7868e4f59179 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-03T17:17:33,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=A 2024-12-03T17:17:33,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:33,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=B 2024-12-03T17:17:33,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:33,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=C 2024-12-03T17:17:33,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:34,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742201_1377 (size=4469) 2024-12-03T17:17:34,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742200_1376 (size=12745) 2024-12-03T17:17:34,050 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86cf735dc0f15035813b7868e4f59179#A#compaction#318 average throughput is 0.24 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:17:34,051 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/8d43c457fb224697a989c5386dc59a60 is 175, key is test_row_0/A:col10/1733246252100/Put/seqid=0 2024-12-03T17:17:34,062 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/9043b1271b744d1ab8343d4bb85b478e as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/9043b1271b744d1ab8343d4bb85b478e 2024-12-03T17:17:34,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. as already flushing 2024-12-03T17:17:34,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:34,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412033c6721b432e141928484171c2466f4bc_86cf735dc0f15035813b7868e4f59179 is 50, key is test_row_0/A:col10/1733246252854/Put/seqid=0 2024-12-03T17:17:34,072 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86cf735dc0f15035813b7868e4f59179/B of 86cf735dc0f15035813b7868e4f59179 into 9043b1271b744d1ab8343d4bb85b478e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:17:34,072 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86cf735dc0f15035813b7868e4f59179: 2024-12-03T17:17:34,072 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., storeName=86cf735dc0f15035813b7868e4f59179/B, priority=13, startTime=1733246253946; duration=0sec 2024-12-03T17:17:34,072 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:17:34,072 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86cf735dc0f15035813b7868e4f59179:B 2024-12-03T17:17:34,072 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:17:34,074 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36945 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:17:34,074 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): 86cf735dc0f15035813b7868e4f59179/C is initiating minor compaction (all files) 2024-12-03T17:17:34,074 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86cf735dc0f15035813b7868e4f59179/C in TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:34,074 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/151107f4668e471586a9f345e5ea4d97, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/fcf9a936139a43b8bc7f6a8932a9ccf2, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/30873cc8521a48e78da1f70c50dcbb6f] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp, totalSize=36.1 K 2024-12-03T17:17:34,075 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 151107f4668e471586a9f345e5ea4d97, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1733246250036 2024-12-03T17:17:34,075 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting fcf9a936139a43b8bc7f6a8932a9ccf2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733246250778 2024-12-03T17:17:34,076 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 30873cc8521a48e78da1f70c50dcbb6f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733246252100 2024-12-03T17:17:34,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742202_1378 (size=31699) 2024-12-03T17:17:34,100 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86cf735dc0f15035813b7868e4f59179#C#compaction#321 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:17:34,101 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/fa90a4c1fd1b4ba09e890375e8eb77f7 is 50, key is test_row_0/C:col10/1733246252100/Put/seqid=0 2024-12-03T17:17:34,130 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/8d43c457fb224697a989c5386dc59a60 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/8d43c457fb224697a989c5386dc59a60 2024-12-03T17:17:34,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742203_1379 (size=12454) 2024-12-03T17:17:34,141 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86cf735dc0f15035813b7868e4f59179/A of 86cf735dc0f15035813b7868e4f59179 into 8d43c457fb224697a989c5386dc59a60(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:17:34,141 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86cf735dc0f15035813b7868e4f59179: 2024-12-03T17:17:34,141 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., storeName=86cf735dc0f15035813b7868e4f59179/A, priority=13, startTime=1733246253923; duration=0sec 2024-12-03T17:17:34,141 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:34,141 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86cf735dc0f15035813b7868e4f59179:A 2024-12-03T17:17:34,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:17:34,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-03T17:17:34,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742204_1380 (size=12745) 2024-12-03T17:17:34,163 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412033c6721b432e141928484171c2466f4bc_86cf735dc0f15035813b7868e4f59179 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412033c6721b432e141928484171c2466f4bc_86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:34,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/9927e8ce134c414bb3e6ea417c33617d, store: [table=TestAcidGuarantees family=A region=86cf735dc0f15035813b7868e4f59179] 2024-12-03T17:17:34,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/9927e8ce134c414bb3e6ea417c33617d is 175, key is test_row_0/A:col10/1733246252854/Put/seqid=0 2024-12-03T17:17:34,177 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/fa90a4c1fd1b4ba09e890375e8eb77f7 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/fa90a4c1fd1b4ba09e890375e8eb77f7 2024-12-03T17:17:34,188 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86cf735dc0f15035813b7868e4f59179/C of 86cf735dc0f15035813b7868e4f59179 into fa90a4c1fd1b4ba09e890375e8eb77f7(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:17:34,189 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86cf735dc0f15035813b7868e4f59179: 2024-12-03T17:17:34,189 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., storeName=86cf735dc0f15035813b7868e4f59179/C, priority=13, startTime=1733246253955; duration=0sec 2024-12-03T17:17:34,189 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:34,189 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86cf735dc0f15035813b7868e4f59179:C 2024-12-03T17:17:34,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742205_1381 (size=31255) 2024-12-03T17:17:34,195 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=288, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/9927e8ce134c414bb3e6ea417c33617d 2024-12-03T17:17:34,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:34,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246314186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:34,197 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:34,197 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:34,197 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:34,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:34,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246314191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:34,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:34,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246314191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:34,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:34,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:34,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:34,223 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:34,223 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:34,223 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:34,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/a4996d290cd046c2958b448f5ff3b8f5 is 50, key is test_row_0/B:col10/1733246252854/Put/seqid=0 2024-12-03T17:17:34,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742206_1382 (size=12301) 2024-12-03T17:17:34,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:34,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246314305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:34,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:34,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:34,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:34,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:34,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246314324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:34,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:34,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:34,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:34,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:34,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246314334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:34,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:34,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:34,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:34,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:34,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246314525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:34,539 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:34,539 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:34,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:34,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:34,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246314544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:34,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:34,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246314544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:34,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:34,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:34,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:34,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:34,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:34,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:34,673 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/a4996d290cd046c2958b448f5ff3b8f5 2024-12-03T17:17:34,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/0b36451b38ee4e399e469bbefe7104ce is 50, key is test_row_0/C:col10/1733246252854/Put/seqid=0 2024-12-03T17:17:34,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742207_1383 (size=12301) 2024-12-03T17:17:34,787 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/0b36451b38ee4e399e469bbefe7104ce 2024-12-03T17:17:34,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/9927e8ce134c414bb3e6ea417c33617d as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/9927e8ce134c414bb3e6ea417c33617d 2024-12-03T17:17:34,835 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/9927e8ce134c414bb3e6ea417c33617d, entries=150, sequenceid=288, filesize=30.5 K 2024-12-03T17:17:34,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/a4996d290cd046c2958b448f5ff3b8f5 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/a4996d290cd046c2958b448f5ff3b8f5 2024-12-03T17:17:34,847 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/a4996d290cd046c2958b448f5ff3b8f5, entries=150, sequenceid=288, filesize=12.0 K 2024-12-03T17:17:34,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/0b36451b38ee4e399e469bbefe7104ce as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/0b36451b38ee4e399e469bbefe7104ce 2024-12-03T17:17:34,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:34,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246314844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:34,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:34,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:34,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:34,867 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/0b36451b38ee4e399e469bbefe7104ce, entries=150, sequenceid=288, filesize=12.0 K 2024-12-03T17:17:34,870 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 86cf735dc0f15035813b7868e4f59179 in 891ms, sequenceid=288, compaction requested=false 2024-12-03T17:17:34,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 86cf735dc0f15035813b7868e4f59179: 2024-12-03T17:17:34,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:34,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-12-03T17:17:34,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-12-03T17:17:34,875 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-12-03T17:17:34,875 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8640 sec 2024-12-03T17:17:34,878 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.8820 sec 2024-12-03T17:17:34,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:34,937 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86cf735dc0f15035813b7868e4f59179 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-03T17:17:34,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=A 2024-12-03T17:17:34,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:34,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=B 2024-12-03T17:17:34,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:34,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=C 2024-12-03T17:17:34,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:34,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412039b25020819f44297bde6c506a635e58a_86cf735dc0f15035813b7868e4f59179 is 50, key is test_row_0/A:col10/1733246254883/Put/seqid=0 2024-12-03T17:17:35,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742208_1384 (size=14994) 2024-12-03T17:17:35,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:35,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246315031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:35,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:35,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:35,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:35,054 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:17:35,063 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412039b25020819f44297bde6c506a635e58a_86cf735dc0f15035813b7868e4f59179 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039b25020819f44297bde6c506a635e58a_86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:35,065 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/e29fa711b0a84596837d43b92bf68e51, store: [table=TestAcidGuarantees family=A region=86cf735dc0f15035813b7868e4f59179] 2024-12-03T17:17:35,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/e29fa711b0a84596837d43b92bf68e51 is 175, key is test_row_0/A:col10/1733246254883/Put/seqid=0 2024-12-03T17:17:35,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:35,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246315052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:35,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:35,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:35,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:35,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742209_1385 (size=39949) 2024-12-03T17:17:35,103 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=309, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/e29fa711b0a84596837d43b92bf68e51 2024-12-03T17:17:35,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/ae8c5b3684c14cf59a8189d9dba33b29 is 50, key is test_row_0/B:col10/1733246254883/Put/seqid=0 2024-12-03T17:17:35,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742210_1386 (size=12301) 2024-12-03T17:17:35,147 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/ae8c5b3684c14cf59a8189d9dba33b29 2024-12-03T17:17:35,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/de62022bdb9c4078bc42bc950e3577d4 is 50, key is test_row_0/C:col10/1733246254883/Put/seqid=0 2024-12-03T17:17:35,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-03T17:17:35,167 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:17:35,168 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T17:17:35,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-12-03T17:17:35,170 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T17:17:35,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-03T17:17:35,170 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T17:17:35,170 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T17:17:35,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:35,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246315167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:35,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:35,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:35,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:35,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:35,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246315187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:35,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:35,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:35,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:35,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742211_1387 (size=12301) 2024-12-03T17:17:35,230 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/de62022bdb9c4078bc42bc950e3577d4 2024-12-03T17:17:35,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/e29fa711b0a84596837d43b92bf68e51 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/e29fa711b0a84596837d43b92bf68e51 2024-12-03T17:17:35,272 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/e29fa711b0a84596837d43b92bf68e51, entries=200, sequenceid=309, filesize=39.0 K 2024-12-03T17:17:35,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/ae8c5b3684c14cf59a8189d9dba33b29 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/ae8c5b3684c14cf59a8189d9dba33b29 2024-12-03T17:17:35,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-03T17:17:35,287 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/ae8c5b3684c14cf59a8189d9dba33b29, entries=150, sequenceid=309, filesize=12.0 K 2024-12-03T17:17:35,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/de62022bdb9c4078bc42bc950e3577d4 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/de62022bdb9c4078bc42bc950e3577d4 2024-12-03T17:17:35,304 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/de62022bdb9c4078bc42bc950e3577d4, entries=150, sequenceid=309, filesize=12.0 K 2024-12-03T17:17:35,305 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 86cf735dc0f15035813b7868e4f59179 in 368ms, sequenceid=309, compaction requested=true 2024-12-03T17:17:35,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86cf735dc0f15035813b7868e4f59179: 2024-12-03T17:17:35,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86cf735dc0f15035813b7868e4f59179:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T17:17:35,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:35,306 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:17:35,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86cf735dc0f15035813b7868e4f59179:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T17:17:35,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:17:35,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86cf735dc0f15035813b7868e4f59179:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T17:17:35,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T17:17:35,306 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:17:35,308 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102903 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:17:35,308 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): 86cf735dc0f15035813b7868e4f59179/A is initiating minor compaction (all files) 2024-12-03T17:17:35,308 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86cf735dc0f15035813b7868e4f59179/A in TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:35,308 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/8d43c457fb224697a989c5386dc59a60, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/9927e8ce134c414bb3e6ea417c33617d, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/e29fa711b0a84596837d43b92bf68e51] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp, totalSize=100.5 K 2024-12-03T17:17:35,308 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:35,308 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. files: [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/8d43c457fb224697a989c5386dc59a60, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/9927e8ce134c414bb3e6ea417c33617d, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/e29fa711b0a84596837d43b92bf68e51] 2024-12-03T17:17:35,309 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:17:35,309 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): 86cf735dc0f15035813b7868e4f59179/B is initiating minor compaction (all files) 2024-12-03T17:17:35,309 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86cf735dc0f15035813b7868e4f59179/B in TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:35,309 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/9043b1271b744d1ab8343d4bb85b478e, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/a4996d290cd046c2958b448f5ff3b8f5, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/ae8c5b3684c14cf59a8189d9dba33b29] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp, totalSize=36.5 K 2024-12-03T17:17:35,310 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d43c457fb224697a989c5386dc59a60, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733246252100 2024-12-03T17:17:35,310 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 9043b1271b744d1ab8343d4bb85b478e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733246252100 2024-12-03T17:17:35,310 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting a4996d290cd046c2958b448f5ff3b8f5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733246252854 2024-12-03T17:17:35,310 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9927e8ce134c414bb3e6ea417c33617d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733246252854 2024-12-03T17:17:35,311 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting ae8c5b3684c14cf59a8189d9dba33b29, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733246254189 2024-12-03T17:17:35,311 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting e29fa711b0a84596837d43b92bf68e51, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733246254160 2024-12-03T17:17:35,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-03T17:17:35,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:35,324 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 86cf735dc0f15035813b7868e4f59179 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-03T17:17:35,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=A 2024-12-03T17:17:35,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:35,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=B 2024-12-03T17:17:35,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:35,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=C 2024-12-03T17:17:35,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:35,340 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86cf735dc0f15035813b7868e4f59179#B#compaction#327 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:17:35,341 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/76e19adcb5de4cdb88ff0c9af5cdef3d is 50, key is test_row_0/B:col10/1733246254883/Put/seqid=0 2024-12-03T17:17:35,355 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=86cf735dc0f15035813b7868e4f59179] 2024-12-03T17:17:35,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412037d50a6ad17c04620bccf72ddfb1d4dd9_86cf735dc0f15035813b7868e4f59179 is 50, key is test_row_0/A:col10/1733246255045/Put/seqid=0 2024-12-03T17:17:35,371 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120301b21e1e3809486aa2a0dc95ef739d8f_86cf735dc0f15035813b7868e4f59179 store=[table=TestAcidGuarantees family=A region=86cf735dc0f15035813b7868e4f59179] 2024-12-03T17:17:35,373 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120301b21e1e3809486aa2a0dc95ef739d8f_86cf735dc0f15035813b7868e4f59179, store=[table=TestAcidGuarantees family=A region=86cf735dc0f15035813b7868e4f59179] 2024-12-03T17:17:35,374 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120301b21e1e3809486aa2a0dc95ef739d8f_86cf735dc0f15035813b7868e4f59179 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=86cf735dc0f15035813b7868e4f59179] 2024-12-03T17:17:35,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. as already flushing 2024-12-03T17:17:35,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:35,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742212_1388 (size=12847) 2024-12-03T17:17:35,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742213_1389 (size=12454) 2024-12-03T17:17:35,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:17:35,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742214_1390 (size=4469) 2024-12-03T17:17:35,446 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86cf735dc0f15035813b7868e4f59179#A#compaction#328 average throughput is 0.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:17:35,447 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/461d3f6e423046be8e2aa197b1f83522 is 175, key is test_row_0/A:col10/1733246254883/Put/seqid=0 2024-12-03T17:17:35,452 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/76e19adcb5de4cdb88ff0c9af5cdef3d as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/76e19adcb5de4cdb88ff0c9af5cdef3d 2024-12-03T17:17:35,458 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412037d50a6ad17c04620bccf72ddfb1d4dd9_86cf735dc0f15035813b7868e4f59179 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412037d50a6ad17c04620bccf72ddfb1d4dd9_86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:35,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/75ab2961ac634997a615b4aacaa39e34, store: [table=TestAcidGuarantees family=A region=86cf735dc0f15035813b7868e4f59179] 2024-12-03T17:17:35,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/75ab2961ac634997a615b4aacaa39e34 is 175, key is test_row_0/A:col10/1733246255045/Put/seqid=0 2024-12-03T17:17:35,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-03T17:17:35,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742215_1391 (size=31801) 2024-12-03T17:17:35,503 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86cf735dc0f15035813b7868e4f59179/B of 86cf735dc0f15035813b7868e4f59179 into 76e19adcb5de4cdb88ff0c9af5cdef3d(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:17:35,503 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86cf735dc0f15035813b7868e4f59179: 2024-12-03T17:17:35,503 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., storeName=86cf735dc0f15035813b7868e4f59179/B, priority=13, startTime=1733246255306; duration=0sec 2024-12-03T17:17:35,503 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:17:35,503 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86cf735dc0f15035813b7868e4f59179:B 2024-12-03T17:17:35,503 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:17:35,511 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/461d3f6e423046be8e2aa197b1f83522 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/461d3f6e423046be8e2aa197b1f83522 2024-12-03T17:17:35,512 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:17:35,512 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): 86cf735dc0f15035813b7868e4f59179/C is initiating minor compaction (all files) 2024-12-03T17:17:35,512 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86cf735dc0f15035813b7868e4f59179/C in TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:35,512 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/fa90a4c1fd1b4ba09e890375e8eb77f7, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/0b36451b38ee4e399e469bbefe7104ce, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/de62022bdb9c4078bc42bc950e3577d4] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp, totalSize=36.5 K 2024-12-03T17:17:35,515 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting fa90a4c1fd1b4ba09e890375e8eb77f7, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733246252100 2024-12-03T17:17:35,516 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b36451b38ee4e399e469bbefe7104ce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733246252854 2024-12-03T17:17:35,519 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting de62022bdb9c4078bc42bc950e3577d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733246254189 2024-12-03T17:17:35,526 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86cf735dc0f15035813b7868e4f59179/A of 86cf735dc0f15035813b7868e4f59179 into 461d3f6e423046be8e2aa197b1f83522(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:17:35,526 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86cf735dc0f15035813b7868e4f59179: 2024-12-03T17:17:35,526 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., storeName=86cf735dc0f15035813b7868e4f59179/A, priority=13, startTime=1733246255306; duration=0sec 2024-12-03T17:17:35,526 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:35,526 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86cf735dc0f15035813b7868e4f59179:A 2024-12-03T17:17:35,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742216_1392 (size=31255) 2024-12-03T17:17:35,579 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86cf735dc0f15035813b7868e4f59179#C#compaction#330 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:17:35,579 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/5c7d1787d14c41c0a8647f211500339d is 50, key is test_row_0/C:col10/1733246254883/Put/seqid=0 2024-12-03T17:17:35,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742217_1393 (size=12847) 2024-12-03T17:17:35,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:35,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246315611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:35,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:35,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:35,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:35,659 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/5c7d1787d14c41c0a8647f211500339d as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/5c7d1787d14c41c0a8647f211500339d 2024-12-03T17:17:35,678 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86cf735dc0f15035813b7868e4f59179/C of 86cf735dc0f15035813b7868e4f59179 into 5c7d1787d14c41c0a8647f211500339d(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:17:35,678 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86cf735dc0f15035813b7868e4f59179: 2024-12-03T17:17:35,678 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., storeName=86cf735dc0f15035813b7868e4f59179/C, priority=13, startTime=1733246255306; duration=0sec 2024-12-03T17:17:35,678 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:35,678 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86cf735dc0f15035813b7868e4f59179:C 2024-12-03T17:17:35,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:35,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246315647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:35,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:35,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:35,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:35,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:35,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246315659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:35,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:35,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:35,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:35,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:35,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246315767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:35,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:35,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:35,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:35,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-03T17:17:35,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:35,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246315798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:35,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:35,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246315808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:35,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:35,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:35,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:35,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:35,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:35,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:35,958 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=323, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/75ab2961ac634997a615b4aacaa39e34 2024-12-03T17:17:35,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/63b37ba86ce042ad8fd6132cfec0054b is 50, key is test_row_0/B:col10/1733246255045/Put/seqid=0 2024-12-03T17:17:36,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:36,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246315994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:36,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:36,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:36,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:36,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742218_1394 (size=12301) 2024-12-03T17:17:36,018 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/63b37ba86ce042ad8fd6132cfec0054b 2024-12-03T17:17:36,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:36,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246316034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:36,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:36,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246316034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:36,046 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:36,046 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:36,046 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:36,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:36,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:36,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:36,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/b26e5881e9d746bd919f327216acfcc5 is 50, key is test_row_0/C:col10/1733246255045/Put/seqid=0 2024-12-03T17:17:36,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742219_1395 (size=12301) 2024-12-03T17:17:36,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-03T17:17:36,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:36,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246316313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:36,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:36,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:36,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:36,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:36,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246316354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:36,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:36,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:36,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:36,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:36,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246316357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:36,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:36,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:36,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:36,492 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/b26e5881e9d746bd919f327216acfcc5 2024-12-03T17:17:36,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/75ab2961ac634997a615b4aacaa39e34 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/75ab2961ac634997a615b4aacaa39e34 2024-12-03T17:17:36,512 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/75ab2961ac634997a615b4aacaa39e34, entries=150, sequenceid=323, filesize=30.5 K 2024-12-03T17:17:36,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/63b37ba86ce042ad8fd6132cfec0054b as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/63b37ba86ce042ad8fd6132cfec0054b 2024-12-03T17:17:36,519 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/63b37ba86ce042ad8fd6132cfec0054b, entries=150, sequenceid=323, filesize=12.0 K 2024-12-03T17:17:36,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/b26e5881e9d746bd919f327216acfcc5 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/b26e5881e9d746bd919f327216acfcc5 2024-12-03T17:17:36,534 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/b26e5881e9d746bd919f327216acfcc5, entries=150, sequenceid=323, filesize=12.0 K 2024-12-03T17:17:36,541 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 86cf735dc0f15035813b7868e4f59179 in 1217ms, sequenceid=323, compaction requested=false 2024-12-03T17:17:36,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 86cf735dc0f15035813b7868e4f59179: 2024-12-03T17:17:36,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:36,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-12-03T17:17:36,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-12-03T17:17:36,549 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-12-03T17:17:36,549 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3760 sec 2024-12-03T17:17:36,555 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.3820 sec 2024-12-03T17:17:36,851 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86cf735dc0f15035813b7868e4f59179 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-03T17:17:36,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=A 2024-12-03T17:17:36,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:36,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=B 2024-12-03T17:17:36,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:36,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=C 2024-12-03T17:17:36,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:36,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:36,885 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ae892f3537b340a4bd6ae5bb960fba5b_86cf735dc0f15035813b7868e4f59179 is 50, key is test_row_0/A:col10/1733246255604/Put/seqid=0 2024-12-03T17:17:36,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742220_1396 (size=12454) 2024-12-03T17:17:36,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:36,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246316933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:36,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:36,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:36,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:36,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:36,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246316936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:36,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:36,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:36,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:36,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:36,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246316960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:36,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:36,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:36,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:37,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:37,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246317064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:37,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:37,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:37,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:37,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:37,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246317064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:37,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:37,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:37,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:37,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:37,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246317085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:37,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:37,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:37,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:37,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:37,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246317284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:37,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:37,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:37,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246317295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:37,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:37,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:37,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:37,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:37,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:37,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:37,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246317313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:37,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-03T17:17:37,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:37,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:37,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:37,315 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:17:37,316 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T17:17:37,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-12-03T17:17:37,321 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T17:17:37,321 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T17:17:37,321 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T17:17:37,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-03T17:17:37,335 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:17:37,341 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ae892f3537b340a4bd6ae5bb960fba5b_86cf735dc0f15035813b7868e4f59179 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ae892f3537b340a4bd6ae5bb960fba5b_86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:37,342 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/7db99f9ba49d48b697a4d573f41a290a, store: [table=TestAcidGuarantees family=A region=86cf735dc0f15035813b7868e4f59179] 2024-12-03T17:17:37,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/7db99f9ba49d48b697a4d573f41a290a is 175, key is test_row_0/A:col10/1733246255604/Put/seqid=0 2024-12-03T17:17:37,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742221_1397 (size=31255) 2024-12-03T17:17:37,387 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=345, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/7db99f9ba49d48b697a4d573f41a290a 2024-12-03T17:17:37,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-03T17:17:37,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/a6a88956372e418b8812d5b11b2d25b0 is 50, key is test_row_0/B:col10/1733246255604/Put/seqid=0 2024-12-03T17:17:37,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742222_1398 (size=12301) 2024-12-03T17:17:37,471 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/a6a88956372e418b8812d5b11b2d25b0 2024-12-03T17:17:37,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-03T17:17:37,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:37,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. as already flushing 2024-12-03T17:17:37,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:37,475 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:37,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:37,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:37,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/24df3e1e5ec0422597d604753594b6c0 is 50, key is test_row_0/C:col10/1733246255604/Put/seqid=0 2024-12-03T17:17:37,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742223_1399 (size=12301) 2024-12-03T17:17:37,563 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/24df3e1e5ec0422597d604753594b6c0 2024-12-03T17:17:37,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/7db99f9ba49d48b697a4d573f41a290a as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/7db99f9ba49d48b697a4d573f41a290a 2024-12-03T17:17:37,599 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/7db99f9ba49d48b697a4d573f41a290a, entries=150, sequenceid=345, filesize=30.5 K 2024-12-03T17:17:37,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/a6a88956372e418b8812d5b11b2d25b0 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/a6a88956372e418b8812d5b11b2d25b0 2024-12-03T17:17:37,604 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/a6a88956372e418b8812d5b11b2d25b0, entries=150, sequenceid=345, filesize=12.0 K 2024-12-03T17:17:37,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/24df3e1e5ec0422597d604753594b6c0 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/24df3e1e5ec0422597d604753594b6c0 2024-12-03T17:17:37,615 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/24df3e1e5ec0422597d604753594b6c0, entries=150, sequenceid=345, filesize=12.0 K 2024-12-03T17:17:37,623 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 86cf735dc0f15035813b7868e4f59179 in 773ms, sequenceid=345, compaction requested=true 2024-12-03T17:17:37,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86cf735dc0f15035813b7868e4f59179: 2024-12-03T17:17:37,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86cf735dc0f15035813b7868e4f59179:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T17:17:37,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:37,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86cf735dc0f15035813b7868e4f59179:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T17:17:37,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:17:37,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86cf735dc0f15035813b7868e4f59179:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T17:17:37,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T17:17:37,623 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:17:37,623 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:17:37,626 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:17:37,626 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): 86cf735dc0f15035813b7868e4f59179/A is initiating minor compaction (all files) 2024-12-03T17:17:37,626 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86cf735dc0f15035813b7868e4f59179/A in TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:37,626 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/461d3f6e423046be8e2aa197b1f83522, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/75ab2961ac634997a615b4aacaa39e34, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/7db99f9ba49d48b697a4d573f41a290a] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp, totalSize=92.1 K 2024-12-03T17:17:37,626 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:37,626 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. files: [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/461d3f6e423046be8e2aa197b1f83522, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/75ab2961ac634997a615b4aacaa39e34, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/7db99f9ba49d48b697a4d573f41a290a] 2024-12-03T17:17:37,627 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 461d3f6e423046be8e2aa197b1f83522, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733246254189 2024-12-03T17:17:37,628 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 75ab2961ac634997a615b4aacaa39e34, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1733246255000 2024-12-03T17:17:37,628 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 7db99f9ba49d48b697a4d573f41a290a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1733246255594 2024-12-03T17:17:37,634 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:17:37,634 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): 86cf735dc0f15035813b7868e4f59179/B is initiating minor compaction (all files) 2024-12-03T17:17:37,634 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86cf735dc0f15035813b7868e4f59179/B in TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:37,634 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/76e19adcb5de4cdb88ff0c9af5cdef3d, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/63b37ba86ce042ad8fd6132cfec0054b, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/a6a88956372e418b8812d5b11b2d25b0] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp, totalSize=36.6 K 2024-12-03T17:17:37,635 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76e19adcb5de4cdb88ff0c9af5cdef3d, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733246254189 2024-12-03T17:17:37,635 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63b37ba86ce042ad8fd6132cfec0054b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1733246255000 2024-12-03T17:17:37,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:37,636 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6a88956372e418b8812d5b11b2d25b0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1733246255594 2024-12-03T17:17:37,638 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86cf735dc0f15035813b7868e4f59179 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-03T17:17:37,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=A 2024-12-03T17:17:37,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:37,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=B 2024-12-03T17:17:37,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:37,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=C 2024-12-03T17:17:37,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:37,643 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=86cf735dc0f15035813b7868e4f59179] 2024-12-03T17:17:37,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-03T17:17:37,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:37,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. as already flushing 2024-12-03T17:17:37,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:37,644 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:37,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:37,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-03T17:17:37,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:37,649 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86cf735dc0f15035813b7868e4f59179#B#compaction#337 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:17:37,650 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/b7beff950ca84e49b6439023eb67af5b is 50, key is test_row_0/B:col10/1733246255604/Put/seqid=0 2024-12-03T17:17:37,676 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412033fdf8bfff65c4ed48c26a8a6c055253b_86cf735dc0f15035813b7868e4f59179 store=[table=TestAcidGuarantees family=A region=86cf735dc0f15035813b7868e4f59179] 2024-12-03T17:17:37,678 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412033fdf8bfff65c4ed48c26a8a6c055253b_86cf735dc0f15035813b7868e4f59179, store=[table=TestAcidGuarantees family=A region=86cf735dc0f15035813b7868e4f59179] 2024-12-03T17:17:37,678 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412033fdf8bfff65c4ed48c26a8a6c055253b_86cf735dc0f15035813b7868e4f59179 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=86cf735dc0f15035813b7868e4f59179] 2024-12-03T17:17:37,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742225_1401 (size=4469) 2024-12-03T17:17:37,718 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86cf735dc0f15035813b7868e4f59179#A#compaction#336 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:17:37,719 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/47e1f664b02941e99ca7323a90fd3361 is 175, key is test_row_0/A:col10/1733246255604/Put/seqid=0 2024-12-03T17:17:37,741 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412037163cc217b1a4c239cd4c7b8ffacb772_86cf735dc0f15035813b7868e4f59179 is 50, key is test_row_0/A:col10/1733246256885/Put/seqid=0 2024-12-03T17:17:37,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742224_1400 (size=12949) 2024-12-03T17:17:37,758 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/b7beff950ca84e49b6439023eb67af5b as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/b7beff950ca84e49b6439023eb67af5b 2024-12-03T17:17:37,763 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86cf735dc0f15035813b7868e4f59179/B of 86cf735dc0f15035813b7868e4f59179 into b7beff950ca84e49b6439023eb67af5b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:17:37,763 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86cf735dc0f15035813b7868e4f59179: 2024-12-03T17:17:37,763 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., storeName=86cf735dc0f15035813b7868e4f59179/B, priority=13, startTime=1733246257623; duration=0sec 2024-12-03T17:17:37,763 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:17:37,763 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86cf735dc0f15035813b7868e4f59179:B 2024-12-03T17:17:37,763 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:17:37,764 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:17:37,764 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): 86cf735dc0f15035813b7868e4f59179/C is initiating minor compaction (all files) 2024-12-03T17:17:37,764 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86cf735dc0f15035813b7868e4f59179/C in TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:37,765 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/5c7d1787d14c41c0a8647f211500339d, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/b26e5881e9d746bd919f327216acfcc5, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/24df3e1e5ec0422597d604753594b6c0] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp, totalSize=36.6 K 2024-12-03T17:17:37,765 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c7d1787d14c41c0a8647f211500339d, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1733246254189 2024-12-03T17:17:37,765 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting b26e5881e9d746bd919f327216acfcc5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1733246255000 2024-12-03T17:17:37,766 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24df3e1e5ec0422597d604753594b6c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1733246255594 2024-12-03T17:17:37,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:37,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246317752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:37,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:37,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:37,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:37,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742226_1402 (size=31903) 2024-12-03T17:17:37,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:37,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246317757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:37,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:37,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246317773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:37,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:37,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:37,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:37,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:37,792 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:37,792 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:37,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-03T17:17:37,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:37,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. as already flushing 2024-12-03T17:17:37,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:37,799 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:37,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:37,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:37,813 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86cf735dc0f15035813b7868e4f59179#C#compaction#339 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:17:37,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742227_1403 (size=14994) 2024-12-03T17:17:37,830 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/d6bb6368b97d4c249a8369550657c8e9 is 50, key is test_row_0/C:col10/1733246255604/Put/seqid=0 2024-12-03T17:17:37,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742228_1404 (size=12949) 2024-12-03T17:17:37,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:37,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246317884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:37,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:37,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:37,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:37,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:37,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246317894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:37,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:37,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246317895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:37,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:37,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:37,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:37,910 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:37,910 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:37,910 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:37,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-03T17:17:37,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-03T17:17:37,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:37,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. as already flushing 2024-12-03T17:17:37,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:37,958 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:37,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:37,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:38,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:38,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246318107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:38,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:38,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:38,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:38,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-03T17:17:38,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:38,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. as already flushing 2024-12-03T17:17:38,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:38,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:38,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:38,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:38,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:38,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246318116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:38,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:38,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246318116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:38,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:38,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:38,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:38,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:38,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:38,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:38,192 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/47e1f664b02941e99ca7323a90fd3361 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/47e1f664b02941e99ca7323a90fd3361 2024-12-03T17:17:38,202 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86cf735dc0f15035813b7868e4f59179/A of 86cf735dc0f15035813b7868e4f59179 into 47e1f664b02941e99ca7323a90fd3361(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:17:38,203 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86cf735dc0f15035813b7868e4f59179: 2024-12-03T17:17:38,203 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., storeName=86cf735dc0f15035813b7868e4f59179/A, priority=13, startTime=1733246257623; duration=0sec 2024-12-03T17:17:38,203 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:38,203 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86cf735dc0f15035813b7868e4f59179:A 2024-12-03T17:17:38,215 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:17:38,233 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412037163cc217b1a4c239cd4c7b8ffacb772_86cf735dc0f15035813b7868e4f59179 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412037163cc217b1a4c239cd4c7b8ffacb772_86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:38,234 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/7b938352b1194b5297b4351141382e9d, store: [table=TestAcidGuarantees family=A region=86cf735dc0f15035813b7868e4f59179] 2024-12-03T17:17:38,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/7b938352b1194b5297b4351141382e9d is 175, key is test_row_0/A:col10/1733246256885/Put/seqid=0 2024-12-03T17:17:38,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742229_1405 (size=39949) 2024-12-03T17:17:38,283 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=359, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/7b938352b1194b5297b4351141382e9d 2024-12-03T17:17:38,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-03T17:17:38,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:38,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. as already flushing 2024-12-03T17:17:38,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:38,295 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:38,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:38,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:38,314 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/d6bb6368b97d4c249a8369550657c8e9 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/d6bb6368b97d4c249a8369550657c8e9 2024-12-03T17:17:38,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/65996f4ac1aa4d10a67b5464b5bae6af is 50, key is test_row_0/B:col10/1733246256885/Put/seqid=0 2024-12-03T17:17:38,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742230_1406 (size=12301) 2024-12-03T17:17:38,367 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86cf735dc0f15035813b7868e4f59179/C of 86cf735dc0f15035813b7868e4f59179 into d6bb6368b97d4c249a8369550657c8e9(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:17:38,367 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86cf735dc0f15035813b7868e4f59179: 2024-12-03T17:17:38,367 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., storeName=86cf735dc0f15035813b7868e4f59179/C, priority=13, startTime=1733246257623; duration=0sec 2024-12-03T17:17:38,367 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:38,367 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86cf735dc0f15035813b7868e4f59179:C 2024-12-03T17:17:38,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-03T17:17:38,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:38,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. as already flushing 2024-12-03T17:17:38,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:38,451 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:38,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:38,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:38,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246318436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:38,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:38,456 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:38,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:38,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:38,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:38,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246318445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:38,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:38,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:38,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:38,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-03T17:17:38,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:38,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246318458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:38,539 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:38,539 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:38,539 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:38,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-03T17:17:38,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:38,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. as already flushing 2024-12-03T17:17:38,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:38,616 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:38,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:38,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:38,758 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/65996f4ac1aa4d10a67b5464b5bae6af 2024-12-03T17:17:38,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-03T17:17:38,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:38,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. as already flushing 2024-12-03T17:17:38,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:38,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:38,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:38,785 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/a0d2ac81df33461083d4c7e1b537cf8b is 50, key is test_row_0/C:col10/1733246256885/Put/seqid=0 2024-12-03T17:17:38,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:38,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742231_1407 (size=12301) 2024-12-03T17:17:38,846 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/a0d2ac81df33461083d4c7e1b537cf8b 2024-12-03T17:17:38,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/7b938352b1194b5297b4351141382e9d as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/7b938352b1194b5297b4351141382e9d 2024-12-03T17:17:38,884 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/7b938352b1194b5297b4351141382e9d, entries=200, sequenceid=359, filesize=39.0 K 2024-12-03T17:17:38,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/65996f4ac1aa4d10a67b5464b5bae6af as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/65996f4ac1aa4d10a67b5464b5bae6af 2024-12-03T17:17:38,889 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/65996f4ac1aa4d10a67b5464b5bae6af, entries=150, sequenceid=359, filesize=12.0 K 2024-12-03T17:17:38,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/a0d2ac81df33461083d4c7e1b537cf8b as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/a0d2ac81df33461083d4c7e1b537cf8b 2024-12-03T17:17:38,894 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/a0d2ac81df33461083d4c7e1b537cf8b, entries=150, sequenceid=359, filesize=12.0 K 2024-12-03T17:17:38,895 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 86cf735dc0f15035813b7868e4f59179 in 1257ms, sequenceid=359, compaction requested=false 2024-12-03T17:17:38,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86cf735dc0f15035813b7868e4f59179: 2024-12-03T17:17:38,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-03T17:17:38,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:38,946 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 86cf735dc0f15035813b7868e4f59179 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-03T17:17:38,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=A 2024-12-03T17:17:38,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:38,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=B 2024-12-03T17:17:38,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:38,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=C 2024-12-03T17:17:38,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:38,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203fc1f05478305483aab146ff342b61d05_86cf735dc0f15035813b7868e4f59179 is 50, key is test_row_0/A:col10/1733246257771/Put/seqid=0 2024-12-03T17:17:38,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. as already flushing 2024-12-03T17:17:38,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:38,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742232_1408 (size=12454) 2024-12-03T17:17:39,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:39,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246319057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:39,067 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:39,067 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:39,067 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:39,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:39,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246319059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:39,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:39,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:39,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:39,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:39,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246319061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:39,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:39,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:39,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:39,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:39,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246319178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:39,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:39,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246319183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:39,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:39,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:39,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:39,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:39,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246319183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:39,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:39,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:39,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:39,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:39,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:39,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:39,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:17:39,398 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203fc1f05478305483aab146ff342b61d05_86cf735dc0f15035813b7868e4f59179 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203fc1f05478305483aab146ff342b61d05_86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:39,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/bae86a572c884b14a1e5e9e8c23b2528, store: [table=TestAcidGuarantees family=A region=86cf735dc0f15035813b7868e4f59179] 2024-12-03T17:17:39,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/bae86a572c884b14a1e5e9e8c23b2528 is 175, key is test_row_0/A:col10/1733246257771/Put/seqid=0 2024-12-03T17:17:39,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:39,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246319405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:39,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:39,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:39,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:39,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:39,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246319410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:39,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:39,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246319410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:39,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:39,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:39,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:39,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:39,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:39,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:39,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742233_1409 (size=31255) 2024-12-03T17:17:39,445 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=380, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/bae86a572c884b14a1e5e9e8c23b2528 2024-12-03T17:17:39,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/e2bd67c3f5544158a4da56389af59320 is 50, key is test_row_0/B:col10/1733246257771/Put/seqid=0 2024-12-03T17:17:39,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742234_1410 (size=12301) 2024-12-03T17:17:39,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-03T17:17:39,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:39,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246319727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:39,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:39,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:39,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:39,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:39,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246319747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:39,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:39,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:39,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:39,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:39,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246319747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:39,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:39,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:39,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:39,931 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/e2bd67c3f5544158a4da56389af59320 2024-12-03T17:17:39,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/4182b97f57da4a29b3981dbd2191467d is 50, key is test_row_0/C:col10/1733246257771/Put/seqid=0 2024-12-03T17:17:39,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742235_1411 (size=12301) 2024-12-03T17:17:39,994 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/4182b97f57da4a29b3981dbd2191467d 2024-12-03T17:17:40,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/bae86a572c884b14a1e5e9e8c23b2528 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/bae86a572c884b14a1e5e9e8c23b2528 2024-12-03T17:17:40,004 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/bae86a572c884b14a1e5e9e8c23b2528, entries=150, sequenceid=380, filesize=30.5 K 2024-12-03T17:17:40,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/e2bd67c3f5544158a4da56389af59320 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/e2bd67c3f5544158a4da56389af59320 2024-12-03T17:17:40,017 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/e2bd67c3f5544158a4da56389af59320, entries=150, sequenceid=380, filesize=12.0 K 2024-12-03T17:17:40,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/4182b97f57da4a29b3981dbd2191467d as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/4182b97f57da4a29b3981dbd2191467d 2024-12-03T17:17:40,033 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/4182b97f57da4a29b3981dbd2191467d, entries=150, sequenceid=380, filesize=12.0 K 2024-12-03T17:17:40,035 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 86cf735dc0f15035813b7868e4f59179 in 1089ms, sequenceid=380, compaction requested=true 2024-12-03T17:17:40,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 86cf735dc0f15035813b7868e4f59179: 2024-12-03T17:17:40,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:40,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-12-03T17:17:40,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-12-03T17:17:40,041 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-12-03T17:17:40,041 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7160 sec 2024-12-03T17:17:40,043 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.7260 sec 2024-12-03T17:17:40,263 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86cf735dc0f15035813b7868e4f59179 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-03T17:17:40,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=A 2024-12-03T17:17:40,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:40,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=B 2024-12-03T17:17:40,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:40,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=C 2024-12-03T17:17:40,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:40,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:40,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203af32cebc89ca4b9a83e7795114d2faf3_86cf735dc0f15035813b7868e4f59179 is 50, key is test_row_0/A:col10/1733246259057/Put/seqid=0 2024-12-03T17:17:40,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742236_1412 (size=14994) 2024-12-03T17:17:40,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:40,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246320351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:40,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:40,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:40,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:40,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:40,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246320366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:40,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:40,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:40,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:40,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:40,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246320366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:40,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:40,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:40,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:40,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:40,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246320475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:40,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:40,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:40,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:40,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:40,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246320493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:40,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:40,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:40,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:40,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:40,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246320493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:40,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:40,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:40,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:40,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:40,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246320695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:40,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:40,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:40,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:40,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:40,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246320725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:40,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:40,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:40,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:40,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:40,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246320726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:40,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:40,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:40,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:40,764 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:17:40,768 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203af32cebc89ca4b9a83e7795114d2faf3_86cf735dc0f15035813b7868e4f59179 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203af32cebc89ca4b9a83e7795114d2faf3_86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:40,769 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/763929724fa042f3b4446b7310b00bcf, store: [table=TestAcidGuarantees family=A region=86cf735dc0f15035813b7868e4f59179] 2024-12-03T17:17:40,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/763929724fa042f3b4446b7310b00bcf is 175, key is test_row_0/A:col10/1733246259057/Put/seqid=0 2024-12-03T17:17:40,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742237_1413 (size=39949) 2024-12-03T17:17:40,785 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=395, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/763929724fa042f3b4446b7310b00bcf 2024-12-03T17:17:40,812 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/3364ffe0b9f146669b0b09de03301452 is 50, key is test_row_0/B:col10/1733246259057/Put/seqid=0 2024-12-03T17:17:40,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742238_1414 (size=12301) 2024-12-03T17:17:41,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:41,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246321025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:41,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:41,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:41,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:41,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:41,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246321046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:41,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:41,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:41,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:41,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:41,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246321063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:41,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:41,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:41,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:41,249 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/3364ffe0b9f146669b0b09de03301452 2024-12-03T17:17:41,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/c284ca2c56684748873e5811588a8a39 is 50, key is test_row_0/C:col10/1733246259057/Put/seqid=0 2024-12-03T17:17:41,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742239_1415 (size=12301) 2024-12-03T17:17:41,358 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/c284ca2c56684748873e5811588a8a39 2024-12-03T17:17:41,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/763929724fa042f3b4446b7310b00bcf as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/763929724fa042f3b4446b7310b00bcf 2024-12-03T17:17:41,371 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/763929724fa042f3b4446b7310b00bcf, entries=200, sequenceid=395, filesize=39.0 K 2024-12-03T17:17:41,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/3364ffe0b9f146669b0b09de03301452 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/3364ffe0b9f146669b0b09de03301452 2024-12-03T17:17:41,391 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/3364ffe0b9f146669b0b09de03301452, entries=150, sequenceid=395, filesize=12.0 K 2024-12-03T17:17:41,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/c284ca2c56684748873e5811588a8a39 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/c284ca2c56684748873e5811588a8a39 2024-12-03T17:17:41,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/c284ca2c56684748873e5811588a8a39, entries=150, sequenceid=395, filesize=12.0 K 2024-12-03T17:17:41,399 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 86cf735dc0f15035813b7868e4f59179 in 1136ms, sequenceid=395, compaction requested=true 2024-12-03T17:17:41,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86cf735dc0f15035813b7868e4f59179: 2024-12-03T17:17:41,400 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T17:17:41,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86cf735dc0f15035813b7868e4f59179:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T17:17:41,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:41,400 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T17:17:41,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86cf735dc0f15035813b7868e4f59179:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T17:17:41,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:41,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86cf735dc0f15035813b7868e4f59179:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T17:17:41,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:17:41,401 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 143056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T17:17:41,401 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): 86cf735dc0f15035813b7868e4f59179/A is initiating minor compaction (all files) 2024-12-03T17:17:41,401 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86cf735dc0f15035813b7868e4f59179/A in TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:41,401 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/47e1f664b02941e99ca7323a90fd3361, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/7b938352b1194b5297b4351141382e9d, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/bae86a572c884b14a1e5e9e8c23b2528, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/763929724fa042f3b4446b7310b00bcf] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp, totalSize=139.7 K 2024-12-03T17:17:41,401 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:41,401 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. files: [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/47e1f664b02941e99ca7323a90fd3361, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/7b938352b1194b5297b4351141382e9d, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/bae86a572c884b14a1e5e9e8c23b2528, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/763929724fa042f3b4446b7310b00bcf] 2024-12-03T17:17:41,402 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 47e1f664b02941e99ca7323a90fd3361, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1733246255594 2024-12-03T17:17:41,402 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T17:17:41,402 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): 86cf735dc0f15035813b7868e4f59179/B is initiating minor compaction (all files) 2024-12-03T17:17:41,402 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86cf735dc0f15035813b7868e4f59179/B in TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:41,403 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/b7beff950ca84e49b6439023eb67af5b, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/65996f4ac1aa4d10a67b5464b5bae6af, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/e2bd67c3f5544158a4da56389af59320, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/3364ffe0b9f146669b0b09de03301452] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp, totalSize=48.7 K 2024-12-03T17:17:41,403 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b938352b1194b5297b4351141382e9d, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1733246256885 2024-12-03T17:17:41,403 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7beff950ca84e49b6439023eb67af5b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1733246255594 2024-12-03T17:17:41,403 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting bae86a572c884b14a1e5e9e8c23b2528, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1733246257710 2024-12-03T17:17:41,403 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65996f4ac1aa4d10a67b5464b5bae6af, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1733246256885 2024-12-03T17:17:41,403 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 763929724fa042f3b4446b7310b00bcf, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1733246259054 2024-12-03T17:17:41,403 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2bd67c3f5544158a4da56389af59320, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1733246257710 2024-12-03T17:17:41,404 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3364ffe0b9f146669b0b09de03301452, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1733246259057 2024-12-03T17:17:41,445 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86cf735dc0f15035813b7868e4f59179#B#compaction#348 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:17:41,446 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/cbe94553ad2142fa845449b03823eb60 is 50, key is test_row_0/B:col10/1733246259057/Put/seqid=0 2024-12-03T17:17:41,464 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=86cf735dc0f15035813b7868e4f59179] 2024-12-03T17:17:41,485 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203d0df60bb429f49c29d8c046db793c37b_86cf735dc0f15035813b7868e4f59179 store=[table=TestAcidGuarantees family=A region=86cf735dc0f15035813b7868e4f59179] 2024-12-03T17:17:41,487 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203d0df60bb429f49c29d8c046db793c37b_86cf735dc0f15035813b7868e4f59179, store=[table=TestAcidGuarantees family=A region=86cf735dc0f15035813b7868e4f59179] 2024-12-03T17:17:41,488 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203d0df60bb429f49c29d8c046db793c37b_86cf735dc0f15035813b7868e4f59179 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=86cf735dc0f15035813b7868e4f59179] 2024-12-03T17:17:41,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742240_1416 (size=13085) 2024-12-03T17:17:41,556 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86cf735dc0f15035813b7868e4f59179 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-03T17:17:41,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=A 2024-12-03T17:17:41,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:41,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=B 2024-12-03T17:17:41,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:41,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=C 2024-12-03T17:17:41,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:41,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:41,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742241_1417 (size=4469) 2024-12-03T17:17:41,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-03T17:17:41,564 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:17:41,565 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T17:17:41,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-12-03T17:17:41,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-03T17:17:41,577 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203a8d43ac557854440a7b155c81caa5f22_86cf735dc0f15035813b7868e4f59179 is 50, key is test_row_0/A:col10/1733246260359/Put/seqid=0 2024-12-03T17:17:41,578 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T17:17:41,586 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T17:17:41,587 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T17:17:41,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742242_1418 (size=14994) 2024-12-03T17:17:41,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-03T17:17:41,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:41,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246321696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:41,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:41,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246321700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:41,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:41,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:41,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:41,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:41,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246321702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:41,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:41,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:41,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:41,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:41,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:41,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:41,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T17:17:41,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:41,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. as already flushing 2024-12-03T17:17:41,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:41,755 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:41,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:41,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:41,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:41,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51304 deadline: 1733246321778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:41,785 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 27e0f852a164,41167,1733246151593 for 'test_row_2' in 86cf735dc0f15035813b7868e4f59179 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18349 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T17:17:41,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:41,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:41,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:41,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:41,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51294 deadline: 1733246321819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:41,834 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 27e0f852a164,41167,1733246151593 for 'test_row_2' in 86cf735dc0f15035813b7868e4f59179 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18408 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T17:17:41,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:41,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:41,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:41,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:41,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246321834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:41,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:41,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246321836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:41,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:41,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246321836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:41,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:41,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:41,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:41,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:41,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:41,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:41,858 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:41,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:41,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:41,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-03T17:17:41,909 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/cbe94553ad2142fa845449b03823eb60 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/cbe94553ad2142fa845449b03823eb60 2024-12-03T17:17:41,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T17:17:41,914 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 86cf735dc0f15035813b7868e4f59179/B of 86cf735dc0f15035813b7868e4f59179 into cbe94553ad2142fa845449b03823eb60(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:17:41,914 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86cf735dc0f15035813b7868e4f59179: 2024-12-03T17:17:41,914 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., storeName=86cf735dc0f15035813b7868e4f59179/B, priority=12, startTime=1733246261400; duration=0sec 2024-12-03T17:17:41,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:41,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. as already flushing 2024-12-03T17:17:41,914 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:17:41,914 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86cf735dc0f15035813b7868e4f59179:B 2024-12-03T17:17:41,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:41,915 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T17:17:41,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:41,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:41,916 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T17:17:41,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:41,916 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): 86cf735dc0f15035813b7868e4f59179/C is initiating minor compaction (all files) 2024-12-03T17:17:41,916 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86cf735dc0f15035813b7868e4f59179/C in TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:41,916 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/d6bb6368b97d4c249a8369550657c8e9, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/a0d2ac81df33461083d4c7e1b537cf8b, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/4182b97f57da4a29b3981dbd2191467d, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/c284ca2c56684748873e5811588a8a39] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp, totalSize=48.7 K 2024-12-03T17:17:41,916 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6bb6368b97d4c249a8369550657c8e9, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1733246255594 2024-12-03T17:17:41,917 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0d2ac81df33461083d4c7e1b537cf8b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1733246256885 2024-12-03T17:17:41,917 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4182b97f57da4a29b3981dbd2191467d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1733246257710 2024-12-03T17:17:41,917 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting c284ca2c56684748873e5811588a8a39, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1733246259057 2024-12-03T17:17:41,932 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86cf735dc0f15035813b7868e4f59179#C#compaction#351 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:17:41,932 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/bd8f9bd6cd634630b092a67314637073 is 50, key is test_row_0/C:col10/1733246259057/Put/seqid=0 2024-12-03T17:17:41,963 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86cf735dc0f15035813b7868e4f59179#A#compaction#349 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:17:41,963 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/c885f14caa934996872eca210ab3f536 is 175, key is test_row_0/A:col10/1733246259057/Put/seqid=0 2024-12-03T17:17:41,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742243_1419 (size=13085) 2024-12-03T17:17:41,984 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/bd8f9bd6cd634630b092a67314637073 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/bd8f9bd6cd634630b092a67314637073 2024-12-03T17:17:41,992 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 86cf735dc0f15035813b7868e4f59179/C of 86cf735dc0f15035813b7868e4f59179 into bd8f9bd6cd634630b092a67314637073(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:17:41,992 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86cf735dc0f15035813b7868e4f59179: 2024-12-03T17:17:41,992 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., storeName=86cf735dc0f15035813b7868e4f59179/C, priority=12, startTime=1733246261401; duration=0sec 2024-12-03T17:17:41,992 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:41,992 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86cf735dc0f15035813b7868e4f59179:C 2024-12-03T17:17:42,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742244_1420 (size=32039) 2024-12-03T17:17:42,035 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:17:42,047 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203a8d43ac557854440a7b155c81caa5f22_86cf735dc0f15035813b7868e4f59179 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203a8d43ac557854440a7b155c81caa5f22_86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:42,048 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/c2ad3c16f594421687f8ea71966f96e4, store: [table=TestAcidGuarantees family=A region=86cf735dc0f15035813b7868e4f59179] 2024-12-03T17:17:42,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/c2ad3c16f594421687f8ea71966f96e4 is 175, key is test_row_0/A:col10/1733246260359/Put/seqid=0 2024-12-03T17:17:42,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T17:17:42,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:42,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246322063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:42,082 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:42,082 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:42,082 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:42,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:42,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246322063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:42,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:42,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:42,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:42,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:42,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:42,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. as already flushing 2024-12-03T17:17:42,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246322063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:42,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:42,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:42,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:42,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:42,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:42,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:42,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:42,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742245_1421 (size=39949) 2024-12-03T17:17:42,101 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=413, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/c2ad3c16f594421687f8ea71966f96e4 2024-12-03T17:17:42,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/cd737afad93446a5b615957a9ca21870 is 50, key is test_row_0/B:col10/1733246260359/Put/seqid=0 2024-12-03T17:17:42,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742246_1422 (size=12301) 2024-12-03T17:17:42,185 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/cd737afad93446a5b615957a9ca21870 2024-12-03T17:17:42,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-03T17:17:42,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T17:17:42,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:42,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. as already flushing 2024-12-03T17:17:42,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:42,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:42,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:42,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:42,249 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/0b07ecc7c5704f509fd2195e5ccc279d is 50, key is test_row_0/C:col10/1733246260359/Put/seqid=0 2024-12-03T17:17:42,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742247_1423 (size=12301) 2024-12-03T17:17:42,280 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/0b07ecc7c5704f509fd2195e5ccc279d 2024-12-03T17:17:42,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/c2ad3c16f594421687f8ea71966f96e4 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/c2ad3c16f594421687f8ea71966f96e4 2024-12-03T17:17:42,305 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/c2ad3c16f594421687f8ea71966f96e4, entries=200, sequenceid=413, filesize=39.0 K 2024-12-03T17:17:42,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/cd737afad93446a5b615957a9ca21870 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/cd737afad93446a5b615957a9ca21870 2024-12-03T17:17:42,325 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/cd737afad93446a5b615957a9ca21870, entries=150, sequenceid=413, filesize=12.0 K 2024-12-03T17:17:42,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/0b07ecc7c5704f509fd2195e5ccc279d as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/0b07ecc7c5704f509fd2195e5ccc279d 2024-12-03T17:17:42,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/0b07ecc7c5704f509fd2195e5ccc279d, entries=150, sequenceid=413, filesize=12.0 K 2024-12-03T17:17:42,338 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 86cf735dc0f15035813b7868e4f59179 in 782ms, sequenceid=413, compaction requested=false 2024-12-03T17:17:42,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86cf735dc0f15035813b7868e4f59179: 2024-12-03T17:17:42,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-03T17:17:42,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:42,399 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 86cf735dc0f15035813b7868e4f59179 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-03T17:17:42,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=A 2024-12-03T17:17:42,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:42,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=B 2024-12-03T17:17:42,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:42,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=C 2024-12-03T17:17:42,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:42,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. as already flushing 2024-12-03T17:17:42,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:42,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120337f73cc98c16409f9096a3f613651e50_86cf735dc0f15035813b7868e4f59179 is 50, key is test_row_0/A:col10/1733246261695/Put/seqid=0 2024-12-03T17:17:42,437 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/c885f14caa934996872eca210ab3f536 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/c885f14caa934996872eca210ab3f536 2024-12-03T17:17:42,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742248_1424 (size=12454) 2024-12-03T17:17:42,455 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 86cf735dc0f15035813b7868e4f59179/A of 86cf735dc0f15035813b7868e4f59179 into c885f14caa934996872eca210ab3f536(size=31.3 K), total size for store is 70.3 K. This selection was in queue for 0sec, and took 1sec to execute. 2024-12-03T17:17:42,455 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86cf735dc0f15035813b7868e4f59179: 2024-12-03T17:17:42,455 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., storeName=86cf735dc0f15035813b7868e4f59179/A, priority=12, startTime=1733246261400; duration=1sec 2024-12-03T17:17:42,455 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:42,455 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86cf735dc0f15035813b7868e4f59179:A 2024-12-03T17:17:42,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:42,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246322520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:42,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:42,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246322523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:42,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:42,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:42,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:42,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:42,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:42,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:42,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:42,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246322538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:42,564 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:42,564 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:42,564 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:42,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:42,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246322655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:42,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:42,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:42,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:42,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246322655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:42,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:42,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:42,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:42,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:42,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:42,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246322674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:42,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:42,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:42,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:42,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-03T17:17:42,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:17:42,858 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120337f73cc98c16409f9096a3f613651e50_86cf735dc0f15035813b7868e4f59179 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120337f73cc98c16409f9096a3f613651e50_86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:42,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/0c772e7ac9914f60a17b0527ef663004, store: [table=TestAcidGuarantees family=A region=86cf735dc0f15035813b7868e4f59179] 2024-12-03T17:17:42,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/0c772e7ac9914f60a17b0527ef663004 is 175, key is test_row_0/A:col10/1733246261695/Put/seqid=0 2024-12-03T17:17:42,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:42,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246322868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:42,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:42,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:42,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:42,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742249_1425 (size=31255) 2024-12-03T17:17:42,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:42,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246322883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:42,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:42,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246322886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:42,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:42,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:42,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:42,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:42,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:42,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:43,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:43,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246323203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:43,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:43,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:43,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:43,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:43,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246323213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:43,232 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:43,232 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:43,232 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:43,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:43,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246323224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:43,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:43,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:43,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:43,296 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=430, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/0c772e7ac9914f60a17b0527ef663004 2024-12-03T17:17:43,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/535606767f9649afaeadb56a41bcb53b is 50, key is test_row_0/B:col10/1733246261695/Put/seqid=0 2024-12-03T17:17:43,381 INFO [Thread-1492 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1492. 2024-12-03T17:17:43,381 DEBUG [Thread-1492 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:17:43,381 DEBUG [Thread-1492 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:43,381 DEBUG [Thread-1492 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:43,381 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:17:43,386 INFO [Thread-1498 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1498. 2024-12-03T17:17:43,386 DEBUG [Thread-1498 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:17:43,386 DEBUG [Thread-1498 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:43,386 DEBUG [Thread-1498 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:43,386 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:17:43,395 INFO [Thread-1494 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1494. 2024-12-03T17:17:43,395 DEBUG [Thread-1494 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:17:43,395 DEBUG [Thread-1494 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:43,396 DEBUG [Thread-1494 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:43,396 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:17:43,396 INFO [Thread-1500 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1500. 2024-12-03T17:17:43,396 DEBUG [Thread-1500 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:17:43,396 DEBUG [Thread-1500 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:43,396 DEBUG [Thread-1500 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:43,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742250_1426 (size=12301) 2024-12-03T17:17:43,406 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:17:43,411 INFO [Thread-1496 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1496. 2024-12-03T17:17:43,411 DEBUG [Thread-1496 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:17:43,411 DEBUG [Thread-1496 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:43,411 DEBUG [Thread-1496 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:43,411 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:17:43,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-03T17:17:43,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:43,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51234 deadline: 1733246323727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:43,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:43,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:43,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:43,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:43,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51238 deadline: 1733246323747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:43,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:43,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51252 deadline: 1733246323747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:43,753 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:43,753 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:43,753 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:43,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:43,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:43,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:17:43,803 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/535606767f9649afaeadb56a41bcb53b 2024-12-03T17:17:43,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/37b2a6800ff44d669de7749fe6aa54ae is 50, key is test_row_0/C:col10/1733246261695/Put/seqid=0 2024-12-03T17:17:43,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742251_1427 (size=12301) 2024-12-03T17:17:43,847 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/37b2a6800ff44d669de7749fe6aa54ae 2024-12-03T17:17:43,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/0c772e7ac9914f60a17b0527ef663004 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/0c772e7ac9914f60a17b0527ef663004 2024-12-03T17:17:43,871 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/0c772e7ac9914f60a17b0527ef663004, entries=150, sequenceid=430, filesize=30.5 K 2024-12-03T17:17:43,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/535606767f9649afaeadb56a41bcb53b as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/535606767f9649afaeadb56a41bcb53b 2024-12-03T17:17:43,909 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/535606767f9649afaeadb56a41bcb53b, entries=150, sequenceid=430, filesize=12.0 K 2024-12-03T17:17:43,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/37b2a6800ff44d669de7749fe6aa54ae as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/37b2a6800ff44d669de7749fe6aa54ae 2024-12-03T17:17:43,922 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/37b2a6800ff44d669de7749fe6aa54ae, entries=150, sequenceid=430, filesize=12.0 K 2024-12-03T17:17:43,923 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 86cf735dc0f15035813b7868e4f59179 in 1524ms, sequenceid=430, compaction requested=true 2024-12-03T17:17:43,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 86cf735dc0f15035813b7868e4f59179: 2024-12-03T17:17:43,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:43,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-12-03T17:17:43,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-12-03T17:17:43,960 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-12-03T17:17:43,960 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3470 sec 2024-12-03T17:17:43,973 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.3980 sec 2024-12-03T17:17:44,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:44,745 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86cf735dc0f15035813b7868e4f59179 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-03T17:17:44,746 INFO [Thread-1485 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1485. 2024-12-03T17:17:44,746 DEBUG [Thread-1485 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:17:44,746 DEBUG [Thread-1485 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:44,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=A 2024-12-03T17:17:44,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:44,746 DEBUG [Thread-1485 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:44,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=B 2024-12-03T17:17:44,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:44,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=C 2024-12-03T17:17:44,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:44,746 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:17:44,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203a9099f9312794effb45a02f543645fb4_86cf735dc0f15035813b7868e4f59179 is 50, key is test_row_0/A:col10/1733246264744/Put/seqid=0 2024-12-03T17:17:44,771 INFO [Thread-1489 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1489. 2024-12-03T17:17:44,771 DEBUG [Thread-1489 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:17:44,771 DEBUG [Thread-1489 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:44,771 DEBUG [Thread-1489 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:44,771 INFO [Thread-1487 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1487. 2024-12-03T17:17:44,771 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:17:44,771 DEBUG [Thread-1487 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:17:44,771 DEBUG [Thread-1487 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:44,771 DEBUG [Thread-1487 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:44,771 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:17:44,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742252_1428 (size=12454) 2024-12-03T17:17:45,207 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:17:45,210 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203a9099f9312794effb45a02f543645fb4_86cf735dc0f15035813b7868e4f59179 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203a9099f9312794effb45a02f543645fb4_86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:45,211 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/a3aa89204eb041ea85b75412310646e1, store: [table=TestAcidGuarantees family=A region=86cf735dc0f15035813b7868e4f59179] 2024-12-03T17:17:45,212 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/a3aa89204eb041ea85b75412310646e1 is 175, key is test_row_0/A:col10/1733246264744/Put/seqid=0 2024-12-03T17:17:45,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742253_1429 (size=31255) 2024-12-03T17:17:45,638 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=449, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/a3aa89204eb041ea85b75412310646e1 2024-12-03T17:17:45,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/779032b08d704ab9a3a939bd1cdf2b1d is 50, key is test_row_0/B:col10/1733246264744/Put/seqid=0 2024-12-03T17:17:45,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742254_1430 (size=12301) 2024-12-03T17:17:45,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-03T17:17:45,754 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:17:46,119 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/779032b08d704ab9a3a939bd1cdf2b1d 2024-12-03T17:17:46,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/01d2b920cb9b452c9c6d3e389170ea76 is 50, key is test_row_0/C:col10/1733246264744/Put/seqid=0 2024-12-03T17:17:46,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742255_1431 (size=12301) 2024-12-03T17:17:46,155 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/01d2b920cb9b452c9c6d3e389170ea76 2024-12-03T17:17:46,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/a3aa89204eb041ea85b75412310646e1 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/a3aa89204eb041ea85b75412310646e1 2024-12-03T17:17:46,182 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/a3aa89204eb041ea85b75412310646e1, entries=150, sequenceid=449, filesize=30.5 K 2024-12-03T17:17:46,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/779032b08d704ab9a3a939bd1cdf2b1d as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/779032b08d704ab9a3a939bd1cdf2b1d 2024-12-03T17:17:46,188 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/779032b08d704ab9a3a939bd1cdf2b1d, entries=150, sequenceid=449, filesize=12.0 K 2024-12-03T17:17:46,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/01d2b920cb9b452c9c6d3e389170ea76 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/01d2b920cb9b452c9c6d3e389170ea76 2024-12-03T17:17:46,204 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/01d2b920cb9b452c9c6d3e389170ea76, entries=150, sequenceid=449, filesize=12.0 K 2024-12-03T17:17:46,205 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=13.42 KB/13740 for 86cf735dc0f15035813b7868e4f59179 in 1460ms, sequenceid=449, compaction requested=true 2024-12-03T17:17:46,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86cf735dc0f15035813b7868e4f59179: 2024-12-03T17:17:46,205 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T17:17:46,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86cf735dc0f15035813b7868e4f59179:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T17:17:46,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:46,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86cf735dc0f15035813b7868e4f59179:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T17:17:46,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:46,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86cf735dc0f15035813b7868e4f59179:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T17:17:46,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:17:46,206 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T17:17:46,212 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134498 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T17:17:46,212 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): 86cf735dc0f15035813b7868e4f59179/A is initiating minor compaction (all files) 2024-12-03T17:17:46,213 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86cf735dc0f15035813b7868e4f59179/A in TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:46,213 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/c885f14caa934996872eca210ab3f536, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/c2ad3c16f594421687f8ea71966f96e4, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/0c772e7ac9914f60a17b0527ef663004, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/a3aa89204eb041ea85b75412310646e1] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp, totalSize=131.3 K 2024-12-03T17:17:46,213 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:46,213 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. files: [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/c885f14caa934996872eca210ab3f536, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/c2ad3c16f594421687f8ea71966f96e4, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/0c772e7ac9914f60a17b0527ef663004, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/a3aa89204eb041ea85b75412310646e1] 2024-12-03T17:17:46,213 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T17:17:46,213 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): 86cf735dc0f15035813b7868e4f59179/B is initiating minor compaction (all files) 2024-12-03T17:17:46,213 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86cf735dc0f15035813b7868e4f59179/B in TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:46,214 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/cbe94553ad2142fa845449b03823eb60, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/cd737afad93446a5b615957a9ca21870, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/535606767f9649afaeadb56a41bcb53b, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/779032b08d704ab9a3a939bd1cdf2b1d] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp, totalSize=48.8 K 2024-12-03T17:17:46,214 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting c885f14caa934996872eca210ab3f536, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1733246259057 2024-12-03T17:17:46,214 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting cbe94553ad2142fa845449b03823eb60, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1733246259057 2024-12-03T17:17:46,214 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2ad3c16f594421687f8ea71966f96e4, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1733246260337 2024-12-03T17:17:46,215 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting cd737afad93446a5b615957a9ca21870, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1733246260359 2024-12-03T17:17:46,215 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c772e7ac9914f60a17b0527ef663004, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1733246261668 2024-12-03T17:17:46,215 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 535606767f9649afaeadb56a41bcb53b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1733246261668 2024-12-03T17:17:46,215 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3aa89204eb041ea85b75412310646e1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1733246262505 2024-12-03T17:17:46,216 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 779032b08d704ab9a3a939bd1cdf2b1d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1733246262505 2024-12-03T17:17:46,254 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86cf735dc0f15035813b7868e4f59179#B#compaction#360 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:17:46,254 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/2ae35bf8c6054f60859bd1a7df59c497 is 50, key is test_row_0/B:col10/1733246264744/Put/seqid=0 2024-12-03T17:17:46,259 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=86cf735dc0f15035813b7868e4f59179] 2024-12-03T17:17:46,279 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241203fd0bebe7a5444805bbe7bf8c73c4d7d3_86cf735dc0f15035813b7868e4f59179 store=[table=TestAcidGuarantees family=A region=86cf735dc0f15035813b7868e4f59179] 2024-12-03T17:17:46,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742256_1432 (size=13221) 2024-12-03T17:17:46,315 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/2ae35bf8c6054f60859bd1a7df59c497 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/2ae35bf8c6054f60859bd1a7df59c497 2024-12-03T17:17:46,320 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241203fd0bebe7a5444805bbe7bf8c73c4d7d3_86cf735dc0f15035813b7868e4f59179, store=[table=TestAcidGuarantees family=A region=86cf735dc0f15035813b7868e4f59179] 2024-12-03T17:17:46,320 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203fd0bebe7a5444805bbe7bf8c73c4d7d3_86cf735dc0f15035813b7868e4f59179 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=86cf735dc0f15035813b7868e4f59179] 2024-12-03T17:17:46,324 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 86cf735dc0f15035813b7868e4f59179/B of 86cf735dc0f15035813b7868e4f59179 into 2ae35bf8c6054f60859bd1a7df59c497(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:17:46,324 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86cf735dc0f15035813b7868e4f59179: 2024-12-03T17:17:46,324 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., storeName=86cf735dc0f15035813b7868e4f59179/B, priority=12, startTime=1733246266206; duration=0sec 2024-12-03T17:17:46,324 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:17:46,324 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86cf735dc0f15035813b7868e4f59179:B 2024-12-03T17:17:46,324 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T17:17:46,326 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T17:17:46,326 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): 86cf735dc0f15035813b7868e4f59179/C is initiating minor compaction (all files) 2024-12-03T17:17:46,326 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86cf735dc0f15035813b7868e4f59179/C in TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:46,326 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/bd8f9bd6cd634630b092a67314637073, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/0b07ecc7c5704f509fd2195e5ccc279d, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/37b2a6800ff44d669de7749fe6aa54ae, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/01d2b920cb9b452c9c6d3e389170ea76] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp, totalSize=48.8 K 2024-12-03T17:17:46,326 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting bd8f9bd6cd634630b092a67314637073, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1733246259057 2024-12-03T17:17:46,331 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b07ecc7c5704f509fd2195e5ccc279d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1733246260359 2024-12-03T17:17:46,331 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 37b2a6800ff44d669de7749fe6aa54ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1733246261668 2024-12-03T17:17:46,331 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 01d2b920cb9b452c9c6d3e389170ea76, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1733246262505 2024-12-03T17:17:46,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742257_1433 (size=4469) 2024-12-03T17:17:46,349 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86cf735dc0f15035813b7868e4f59179#A#compaction#361 average throughput is 0.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:17:46,351 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/53a95a6be8154cb7b2998d6723872d49 is 175, key is test_row_0/A:col10/1733246264744/Put/seqid=0 2024-12-03T17:17:46,382 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86cf735dc0f15035813b7868e4f59179#C#compaction#362 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:17:46,383 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/6ee16d7aa69c4a0c8c0ad7afa7041c53 is 50, key is test_row_0/C:col10/1733246264744/Put/seqid=0 2024-12-03T17:17:46,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742258_1434 (size=32175) 2024-12-03T17:17:46,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742259_1435 (size=13221) 2024-12-03T17:17:46,815 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/6ee16d7aa69c4a0c8c0ad7afa7041c53 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/6ee16d7aa69c4a0c8c0ad7afa7041c53 2024-12-03T17:17:46,820 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/53a95a6be8154cb7b2998d6723872d49 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/53a95a6be8154cb7b2998d6723872d49 2024-12-03T17:17:46,824 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 86cf735dc0f15035813b7868e4f59179/C of 86cf735dc0f15035813b7868e4f59179 into 6ee16d7aa69c4a0c8c0ad7afa7041c53(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:17:46,824 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86cf735dc0f15035813b7868e4f59179: 2024-12-03T17:17:46,824 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., storeName=86cf735dc0f15035813b7868e4f59179/C, priority=12, startTime=1733246266206; duration=0sec 2024-12-03T17:17:46,825 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:46,825 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86cf735dc0f15035813b7868e4f59179:C 2024-12-03T17:17:46,828 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 86cf735dc0f15035813b7868e4f59179/A of 86cf735dc0f15035813b7868e4f59179 into 53a95a6be8154cb7b2998d6723872d49(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:17:46,828 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86cf735dc0f15035813b7868e4f59179: 2024-12-03T17:17:46,828 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., storeName=86cf735dc0f15035813b7868e4f59179/A, priority=12, startTime=1733246266205; duration=0sec 2024-12-03T17:17:46,828 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:17:46,828 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86cf735dc0f15035813b7868e4f59179:A 2024-12-03T17:17:49,400 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T17:17:51,815 INFO [Thread-1483 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1483. 2024-12-03T17:17:51,815 DEBUG [Thread-1483 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:17:51,816 DEBUG [Thread-1483 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:51,816 DEBUG [Thread-1483 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:51,816 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:17:51,919 INFO [Thread-1481 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1481. 2024-12-03T17:17:51,919 DEBUG [Thread-1481 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:17:51,919 DEBUG [Thread-1481 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:51,920 DEBUG [Thread-1481 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:51,920 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:17:51,920 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-03T17:17:51,920 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 4 2024-12-03T17:17:51,920 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 5 2024-12-03T17:17:51,920 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 122 2024-12-03T17:17:51,920 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 111 2024-12-03T17:17:51,920 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 126 2024-12-03T17:17:51,920 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-03T17:17:51,920 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-03T17:17:51,920 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 636 2024-12-03T17:17:51,920 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1906 rows 2024-12-03T17:17:51,920 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 642 2024-12-03T17:17:51,920 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1926 rows 2024-12-03T17:17:51,920 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 652 2024-12-03T17:17:51,920 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1956 rows 2024-12-03T17:17:51,920 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 650 2024-12-03T17:17:51,920 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1950 rows 2024-12-03T17:17:51,920 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 648 2024-12-03T17:17:51,920 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1944 rows 2024-12-03T17:17:51,920 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T17:17:51,920 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T17:17:51,920 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:51,921 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:51,921 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:17:51,921 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-03T17:17:51,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-03T17:17:51,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-03T17:17:51,925 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733246271925"}]},"ts":"1733246271925"} 2024-12-03T17:17:51,927 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-03T17:17:51,927 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-03T17:17:51,928 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T17:17:51,934 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=86cf735dc0f15035813b7868e4f59179, UNASSIGN}] 2024-12-03T17:17:51,935 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=86cf735dc0f15035813b7868e4f59179, UNASSIGN 2024-12-03T17:17:51,936 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=107 updating hbase:meta row=86cf735dc0f15035813b7868e4f59179, regionState=CLOSING, regionLocation=27e0f852a164,41167,1733246151593 2024-12-03T17:17:51,938 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=86cf735dc0f15035813b7868e4f59179, UNASSIGN because future has completed 2024-12-03T17:17:51,938 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T17:17:51,938 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; CloseRegionProcedure 86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593}] 2024-12-03T17:17:52,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-03T17:17:52,091 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] handler.UnassignRegionHandler(122): Close 86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:52,091 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T17:17:52,091 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1722): Closing 86cf735dc0f15035813b7868e4f59179, disabling compactions & flushes 2024-12-03T17:17:52,091 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:52,091 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:52,091 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. after waiting 0 ms 2024-12-03T17:17:52,091 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:52,091 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(2902): Flushing 86cf735dc0f15035813b7868e4f59179 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-03T17:17:52,092 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=A 2024-12-03T17:17:52,092 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:52,092 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=B 2024-12-03T17:17:52,092 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:52,092 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86cf735dc0f15035813b7868e4f59179, store=C 2024-12-03T17:17:52,092 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:52,102 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412039c0a62c4598b4f748152ef7add32f401_86cf735dc0f15035813b7868e4f59179 is 50, key is test_row_0/A:col10/1733246264764/Put/seqid=0 2024-12-03T17:17:52,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742260_1436 (size=12454) 2024-12-03T17:17:52,131 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:17:52,145 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412039c0a62c4598b4f748152ef7add32f401_86cf735dc0f15035813b7868e4f59179 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039c0a62c4598b4f748152ef7add32f401_86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:52,155 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/038c3ac7cba04c7cb87e53a8f9af9c2a, store: [table=TestAcidGuarantees family=A region=86cf735dc0f15035813b7868e4f59179] 2024-12-03T17:17:52,156 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/038c3ac7cba04c7cb87e53a8f9af9c2a is 175, key is test_row_0/A:col10/1733246264764/Put/seqid=0 2024-12-03T17:17:52,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742261_1437 (size=31255) 2024-12-03T17:17:52,180 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=459, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/038c3ac7cba04c7cb87e53a8f9af9c2a 2024-12-03T17:17:52,202 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/3191fac033314322bc6649406d1e314a is 50, key is test_row_0/B:col10/1733246264764/Put/seqid=0 2024-12-03T17:17:52,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-03T17:17:52,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742262_1438 (size=12301) 2024-12-03T17:17:52,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-03T17:17:52,660 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/3191fac033314322bc6649406d1e314a 2024-12-03T17:17:52,719 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/e96eabd5b39f4e4e9cd28740506c0b2b is 50, key is test_row_0/C:col10/1733246264764/Put/seqid=0 2024-12-03T17:17:52,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742263_1439 (size=12301) 2024-12-03T17:17:52,750 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/e96eabd5b39f4e4e9cd28740506c0b2b 2024-12-03T17:17:52,761 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/A/038c3ac7cba04c7cb87e53a8f9af9c2a as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/038c3ac7cba04c7cb87e53a8f9af9c2a 2024-12-03T17:17:52,772 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/038c3ac7cba04c7cb87e53a8f9af9c2a, entries=150, sequenceid=459, filesize=30.5 K 2024-12-03T17:17:52,773 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/B/3191fac033314322bc6649406d1e314a as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/3191fac033314322bc6649406d1e314a 2024-12-03T17:17:52,780 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/3191fac033314322bc6649406d1e314a, entries=150, sequenceid=459, filesize=12.0 K 2024-12-03T17:17:52,781 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/.tmp/C/e96eabd5b39f4e4e9cd28740506c0b2b as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/e96eabd5b39f4e4e9cd28740506c0b2b 2024-12-03T17:17:52,797 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/e96eabd5b39f4e4e9cd28740506c0b2b, entries=150, sequenceid=459, filesize=12.0 K 2024-12-03T17:17:52,803 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 86cf735dc0f15035813b7868e4f59179 in 712ms, sequenceid=459, compaction requested=false 2024-12-03T17:17:52,815 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/48e4f5cae8f848a9aa711765b4bd4184, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/80b8a8f2d55041d5af075f0e947512e1, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/a0bc32c6219f4f7eb594eeff575375af, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/04ec9678e612435886d5866904f507ff, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/ed1678830d1e4900ad8bf316fbd70150, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/a669714aaae046a8a5efddd96c9098df, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/0552189a8d044c18900cf3ab0dc43d23, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/07ca3656577e4ede8607f9502946da14, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/21f739896e614231b7d90b92b9762104, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/fde5c62620934e9886ef6c08f59369d9, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/81ba4302f14f487891f64fee38f3a05d, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/701ebf0ef67a43eebe8536582d86c300, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/b168b6771648432ead8f092758847443, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/8d43c457fb224697a989c5386dc59a60, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/9927e8ce134c414bb3e6ea417c33617d, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/e29fa711b0a84596837d43b92bf68e51, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/461d3f6e423046be8e2aa197b1f83522, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/75ab2961ac634997a615b4aacaa39e34, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/47e1f664b02941e99ca7323a90fd3361, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/7db99f9ba49d48b697a4d573f41a290a, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/7b938352b1194b5297b4351141382e9d, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/bae86a572c884b14a1e5e9e8c23b2528, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/763929724fa042f3b4446b7310b00bcf, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/c885f14caa934996872eca210ab3f536, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/c2ad3c16f594421687f8ea71966f96e4, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/0c772e7ac9914f60a17b0527ef663004, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/a3aa89204eb041ea85b75412310646e1] to archive 2024-12-03T17:17:52,816 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T17:17:52,824 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/48e4f5cae8f848a9aa711765b4bd4184 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/48e4f5cae8f848a9aa711765b4bd4184 2024-12-03T17:17:52,828 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/80b8a8f2d55041d5af075f0e947512e1 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/80b8a8f2d55041d5af075f0e947512e1 2024-12-03T17:17:52,843 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/a0bc32c6219f4f7eb594eeff575375af to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/a0bc32c6219f4f7eb594eeff575375af 2024-12-03T17:17:52,854 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/04ec9678e612435886d5866904f507ff to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/04ec9678e612435886d5866904f507ff 2024-12-03T17:17:52,866 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/ed1678830d1e4900ad8bf316fbd70150 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/ed1678830d1e4900ad8bf316fbd70150 2024-12-03T17:17:52,872 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/a669714aaae046a8a5efddd96c9098df to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/a669714aaae046a8a5efddd96c9098df 2024-12-03T17:17:52,874 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/0552189a8d044c18900cf3ab0dc43d23 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/0552189a8d044c18900cf3ab0dc43d23 2024-12-03T17:17:52,884 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/07ca3656577e4ede8607f9502946da14 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/07ca3656577e4ede8607f9502946da14 2024-12-03T17:17:52,886 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/21f739896e614231b7d90b92b9762104 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/21f739896e614231b7d90b92b9762104 2024-12-03T17:17:52,894 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/fde5c62620934e9886ef6c08f59369d9 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/fde5c62620934e9886ef6c08f59369d9 2024-12-03T17:17:52,897 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/81ba4302f14f487891f64fee38f3a05d to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/81ba4302f14f487891f64fee38f3a05d 2024-12-03T17:17:52,900 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/701ebf0ef67a43eebe8536582d86c300 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/701ebf0ef67a43eebe8536582d86c300 2024-12-03T17:17:52,902 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/b168b6771648432ead8f092758847443 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/b168b6771648432ead8f092758847443 2024-12-03T17:17:52,903 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/8d43c457fb224697a989c5386dc59a60 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/8d43c457fb224697a989c5386dc59a60 2024-12-03T17:17:52,926 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/9927e8ce134c414bb3e6ea417c33617d to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/9927e8ce134c414bb3e6ea417c33617d 2024-12-03T17:17:52,929 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/e29fa711b0a84596837d43b92bf68e51 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/e29fa711b0a84596837d43b92bf68e51 2024-12-03T17:17:52,930 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/461d3f6e423046be8e2aa197b1f83522 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/461d3f6e423046be8e2aa197b1f83522 2024-12-03T17:17:52,933 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/75ab2961ac634997a615b4aacaa39e34 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/75ab2961ac634997a615b4aacaa39e34 2024-12-03T17:17:52,934 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/47e1f664b02941e99ca7323a90fd3361 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/47e1f664b02941e99ca7323a90fd3361 2024-12-03T17:17:52,936 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/7db99f9ba49d48b697a4d573f41a290a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/7db99f9ba49d48b697a4d573f41a290a 2024-12-03T17:17:52,937 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/7b938352b1194b5297b4351141382e9d to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/7b938352b1194b5297b4351141382e9d 2024-12-03T17:17:52,939 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/bae86a572c884b14a1e5e9e8c23b2528 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/bae86a572c884b14a1e5e9e8c23b2528 2024-12-03T17:17:52,940 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/763929724fa042f3b4446b7310b00bcf to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/763929724fa042f3b4446b7310b00bcf 2024-12-03T17:17:52,943 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/c885f14caa934996872eca210ab3f536 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/c885f14caa934996872eca210ab3f536 2024-12-03T17:17:52,956 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/c2ad3c16f594421687f8ea71966f96e4 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/c2ad3c16f594421687f8ea71966f96e4 2024-12-03T17:17:52,960 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/0c772e7ac9914f60a17b0527ef663004 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/0c772e7ac9914f60a17b0527ef663004 2024-12-03T17:17:52,961 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/a3aa89204eb041ea85b75412310646e1 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/a3aa89204eb041ea85b75412310646e1 2024-12-03T17:17:52,965 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/c320b220742c4ae7937322c33eda32a7, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/4cf4ff6088a245deadaf3f3df91b5c03, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/fac0875a75ed4c7a91cc6561917b2be8, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/6eed8cd057614c768e550d8034cfd2ff, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/954c951e23214cfe908a22934920f6bc, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/82e117a1199d47b0b30568724b7ace4c, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/54191cdcf9134e2d92e78800ba0903b0, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/005d3848b3884d47957bbb6fb6950e7f, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/49704ba960854de381fabd58c030851e, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/c570211cfe0b4e3896d54bd981ca7fba, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/594a9894afe34f77965b9d859b79d52a, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/44ce70fb0c774073bef4639afed0465e, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/9043b1271b744d1ab8343d4bb85b478e, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/ae77c1dc366d47cd81677d176f9d20f0, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/a4996d290cd046c2958b448f5ff3b8f5, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/76e19adcb5de4cdb88ff0c9af5cdef3d, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/ae8c5b3684c14cf59a8189d9dba33b29, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/63b37ba86ce042ad8fd6132cfec0054b, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/b7beff950ca84e49b6439023eb67af5b, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/a6a88956372e418b8812d5b11b2d25b0, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/65996f4ac1aa4d10a67b5464b5bae6af, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/e2bd67c3f5544158a4da56389af59320, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/cbe94553ad2142fa845449b03823eb60, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/3364ffe0b9f146669b0b09de03301452, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/cd737afad93446a5b615957a9ca21870, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/535606767f9649afaeadb56a41bcb53b, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/779032b08d704ab9a3a939bd1cdf2b1d] to archive 2024-12-03T17:17:52,966 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T17:17:52,970 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/c320b220742c4ae7937322c33eda32a7 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/c320b220742c4ae7937322c33eda32a7 2024-12-03T17:17:52,974 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/4cf4ff6088a245deadaf3f3df91b5c03 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/4cf4ff6088a245deadaf3f3df91b5c03 2024-12-03T17:17:52,978 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/fac0875a75ed4c7a91cc6561917b2be8 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/fac0875a75ed4c7a91cc6561917b2be8 2024-12-03T17:17:52,981 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/6eed8cd057614c768e550d8034cfd2ff to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/6eed8cd057614c768e550d8034cfd2ff 2024-12-03T17:17:52,984 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/954c951e23214cfe908a22934920f6bc to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/954c951e23214cfe908a22934920f6bc 2024-12-03T17:17:52,987 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/82e117a1199d47b0b30568724b7ace4c to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/82e117a1199d47b0b30568724b7ace4c 2024-12-03T17:17:52,991 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/54191cdcf9134e2d92e78800ba0903b0 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/54191cdcf9134e2d92e78800ba0903b0 2024-12-03T17:17:52,998 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/005d3848b3884d47957bbb6fb6950e7f to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/005d3848b3884d47957bbb6fb6950e7f 2024-12-03T17:17:53,001 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/49704ba960854de381fabd58c030851e to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/49704ba960854de381fabd58c030851e 2024-12-03T17:17:53,004 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/c570211cfe0b4e3896d54bd981ca7fba to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/c570211cfe0b4e3896d54bd981ca7fba 2024-12-03T17:17:53,006 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/594a9894afe34f77965b9d859b79d52a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/594a9894afe34f77965b9d859b79d52a 2024-12-03T17:17:53,008 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/44ce70fb0c774073bef4639afed0465e to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/44ce70fb0c774073bef4639afed0465e 2024-12-03T17:17:53,010 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/9043b1271b744d1ab8343d4bb85b478e to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/9043b1271b744d1ab8343d4bb85b478e 2024-12-03T17:17:53,019 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/ae77c1dc366d47cd81677d176f9d20f0 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/ae77c1dc366d47cd81677d176f9d20f0 2024-12-03T17:17:53,021 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/a4996d290cd046c2958b448f5ff3b8f5 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/a4996d290cd046c2958b448f5ff3b8f5 2024-12-03T17:17:53,024 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/76e19adcb5de4cdb88ff0c9af5cdef3d to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/76e19adcb5de4cdb88ff0c9af5cdef3d 2024-12-03T17:17:53,026 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/ae8c5b3684c14cf59a8189d9dba33b29 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/ae8c5b3684c14cf59a8189d9dba33b29 2024-12-03T17:17:53,028 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/63b37ba86ce042ad8fd6132cfec0054b to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/63b37ba86ce042ad8fd6132cfec0054b 2024-12-03T17:17:53,029 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/b7beff950ca84e49b6439023eb67af5b to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/b7beff950ca84e49b6439023eb67af5b 2024-12-03T17:17:53,033 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/a6a88956372e418b8812d5b11b2d25b0 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/a6a88956372e418b8812d5b11b2d25b0 2024-12-03T17:17:53,035 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/65996f4ac1aa4d10a67b5464b5bae6af to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/65996f4ac1aa4d10a67b5464b5bae6af 2024-12-03T17:17:53,037 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/e2bd67c3f5544158a4da56389af59320 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/e2bd67c3f5544158a4da56389af59320 2024-12-03T17:17:53,041 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/cbe94553ad2142fa845449b03823eb60 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/cbe94553ad2142fa845449b03823eb60 2024-12-03T17:17:53,046 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/3364ffe0b9f146669b0b09de03301452 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/3364ffe0b9f146669b0b09de03301452 2024-12-03T17:17:53,048 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/cd737afad93446a5b615957a9ca21870 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/cd737afad93446a5b615957a9ca21870 2024-12-03T17:17:53,051 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/535606767f9649afaeadb56a41bcb53b to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/535606767f9649afaeadb56a41bcb53b 2024-12-03T17:17:53,057 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/779032b08d704ab9a3a939bd1cdf2b1d to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/779032b08d704ab9a3a939bd1cdf2b1d 2024-12-03T17:17:53,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-03T17:17:53,071 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/97ccc85f920d49a085a0acf4e0585131, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/eae44ee6df8744ee8e77cec1545cc69d, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/89c68435e1cf473492410d7aaf6dff79, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/9ef60ee4b00f4ace9cde568066251671, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/08690ce989124676ad79c8de07ec0e41, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/59711e2f42ca4a499ec356e07545bd78, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/b90a5cd0e6c94fbfbbc093f711e272cf, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/42a218214b09455c871bbc056c159ba7, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/29fe3c384e67415298a897fc6401babe, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/151107f4668e471586a9f345e5ea4d97, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/483026c942b04cb0b760489a4c51ab7f, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/fcf9a936139a43b8bc7f6a8932a9ccf2, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/fa90a4c1fd1b4ba09e890375e8eb77f7, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/30873cc8521a48e78da1f70c50dcbb6f, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/0b36451b38ee4e399e469bbefe7104ce, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/5c7d1787d14c41c0a8647f211500339d, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/de62022bdb9c4078bc42bc950e3577d4, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/b26e5881e9d746bd919f327216acfcc5, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/d6bb6368b97d4c249a8369550657c8e9, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/24df3e1e5ec0422597d604753594b6c0, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/a0d2ac81df33461083d4c7e1b537cf8b, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/4182b97f57da4a29b3981dbd2191467d, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/bd8f9bd6cd634630b092a67314637073, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/c284ca2c56684748873e5811588a8a39, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/0b07ecc7c5704f509fd2195e5ccc279d, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/37b2a6800ff44d669de7749fe6aa54ae, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/01d2b920cb9b452c9c6d3e389170ea76] to archive 2024-12-03T17:17:53,078 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T17:17:53,084 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/97ccc85f920d49a085a0acf4e0585131 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/97ccc85f920d49a085a0acf4e0585131 2024-12-03T17:17:53,088 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/eae44ee6df8744ee8e77cec1545cc69d to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/eae44ee6df8744ee8e77cec1545cc69d 2024-12-03T17:17:53,091 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/89c68435e1cf473492410d7aaf6dff79 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/89c68435e1cf473492410d7aaf6dff79 2024-12-03T17:17:53,098 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/9ef60ee4b00f4ace9cde568066251671 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/9ef60ee4b00f4ace9cde568066251671 2024-12-03T17:17:53,102 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/08690ce989124676ad79c8de07ec0e41 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/08690ce989124676ad79c8de07ec0e41 2024-12-03T17:17:53,104 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/59711e2f42ca4a499ec356e07545bd78 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/59711e2f42ca4a499ec356e07545bd78 2024-12-03T17:17:53,109 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/b90a5cd0e6c94fbfbbc093f711e272cf to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/b90a5cd0e6c94fbfbbc093f711e272cf 2024-12-03T17:17:53,124 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/42a218214b09455c871bbc056c159ba7 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/42a218214b09455c871bbc056c159ba7 2024-12-03T17:17:53,126 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/29fe3c384e67415298a897fc6401babe to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/29fe3c384e67415298a897fc6401babe 2024-12-03T17:17:53,127 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/151107f4668e471586a9f345e5ea4d97 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/151107f4668e471586a9f345e5ea4d97 2024-12-03T17:17:53,129 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/483026c942b04cb0b760489a4c51ab7f to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/483026c942b04cb0b760489a4c51ab7f 2024-12-03T17:17:53,132 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/fcf9a936139a43b8bc7f6a8932a9ccf2 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/fcf9a936139a43b8bc7f6a8932a9ccf2 2024-12-03T17:17:53,133 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/fa90a4c1fd1b4ba09e890375e8eb77f7 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/fa90a4c1fd1b4ba09e890375e8eb77f7 2024-12-03T17:17:53,135 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/30873cc8521a48e78da1f70c50dcbb6f to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/30873cc8521a48e78da1f70c50dcbb6f 2024-12-03T17:17:53,136 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/0b36451b38ee4e399e469bbefe7104ce to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/0b36451b38ee4e399e469bbefe7104ce 2024-12-03T17:17:53,137 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/5c7d1787d14c41c0a8647f211500339d to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/5c7d1787d14c41c0a8647f211500339d 2024-12-03T17:17:53,139 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/de62022bdb9c4078bc42bc950e3577d4 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/de62022bdb9c4078bc42bc950e3577d4 2024-12-03T17:17:53,141 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/b26e5881e9d746bd919f327216acfcc5 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/b26e5881e9d746bd919f327216acfcc5 2024-12-03T17:17:53,142 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/d6bb6368b97d4c249a8369550657c8e9 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/d6bb6368b97d4c249a8369550657c8e9 2024-12-03T17:17:53,148 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/24df3e1e5ec0422597d604753594b6c0 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/24df3e1e5ec0422597d604753594b6c0 2024-12-03T17:17:53,149 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/a0d2ac81df33461083d4c7e1b537cf8b to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/a0d2ac81df33461083d4c7e1b537cf8b 2024-12-03T17:17:53,151 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/4182b97f57da4a29b3981dbd2191467d to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/4182b97f57da4a29b3981dbd2191467d 2024-12-03T17:17:53,156 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/bd8f9bd6cd634630b092a67314637073 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/bd8f9bd6cd634630b092a67314637073 2024-12-03T17:17:53,158 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/c284ca2c56684748873e5811588a8a39 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/c284ca2c56684748873e5811588a8a39 2024-12-03T17:17:53,160 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/0b07ecc7c5704f509fd2195e5ccc279d to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/0b07ecc7c5704f509fd2195e5ccc279d 2024-12-03T17:17:53,161 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/37b2a6800ff44d669de7749fe6aa54ae to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/37b2a6800ff44d669de7749fe6aa54ae 2024-12-03T17:17:53,167 DEBUG [StoreCloser-TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/01d2b920cb9b452c9c6d3e389170ea76 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/01d2b920cb9b452c9c6d3e389170ea76 2024-12-03T17:17:53,233 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/recovered.edits/462.seqid, newMaxSeqId=462, maxSeqId=4 2024-12-03T17:17:53,241 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. 2024-12-03T17:17:53,241 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1676): Region close journal for 86cf735dc0f15035813b7868e4f59179: Waiting for close lock at 1733246272091Running coprocessor pre-close hooks at 1733246272091Disabling compacts and flushes for region at 1733246272091Disabling writes for close at 1733246272091Obtaining lock to block concurrent updates at 1733246272092 (+1 ms)Preparing flush snapshotting stores in 86cf735dc0f15035813b7868e4f59179 at 1733246272092Finished memstore snapshotting TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733246272092Flushing stores of TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179. at 1733246272093 (+1 ms)Flushing 86cf735dc0f15035813b7868e4f59179/A: creating writer at 1733246272093Flushing mob file 86cf735dc0f15035813b7868e4f59179/A: appending metadata at 1733246272102 (+9 ms)Flushing mob file 86cf735dc0f15035813b7868e4f59179/A: closing flushed file at 1733246272102Flushing 86cf735dc0f15035813b7868e4f59179/A: appending metadata at 1733246272155 (+53 ms)Flushing 86cf735dc0f15035813b7868e4f59179/A: closing flushed file at 1733246272155Flushing 86cf735dc0f15035813b7868e4f59179/B: creating writer at 1733246272194 (+39 ms)Flushing 86cf735dc0f15035813b7868e4f59179/B: appending metadata at 1733246272201 (+7 ms)Flushing 86cf735dc0f15035813b7868e4f59179/B: closing flushed file at 1733246272201Flushing 86cf735dc0f15035813b7868e4f59179/C: creating writer at 1733246272699 (+498 ms)Flushing 86cf735dc0f15035813b7868e4f59179/C: appending metadata at 1733246272718 (+19 ms)Flushing 86cf735dc0f15035813b7868e4f59179/C: closing flushed file at 1733246272718Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2897ad02: reopening flushed file at 1733246272759 (+41 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@820a36c: reopening flushed file at 1733246272772 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@45236570: reopening flushed file at 1733246272780 (+8 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 86cf735dc0f15035813b7868e4f59179 in 712ms, sequenceid=459, compaction requested=false at 1733246272803 (+23 ms)Writing region close event to WAL at 1733246273186 (+383 ms)Running coprocessor post-close hooks at 1733246273241 (+55 ms)Closed at 1733246273241 2024-12-03T17:17:53,245 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=107 updating hbase:meta row=86cf735dc0f15035813b7868e4f59179, regionState=CLOSED 2024-12-03T17:17:53,247 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] handler.UnassignRegionHandler(157): Closed 86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:53,256 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=108, ppid=107, state=RUNNABLE, hasLock=false; CloseRegionProcedure 86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 because future has completed 2024-12-03T17:17:53,280 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-12-03T17:17:53,281 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; CloseRegionProcedure 86cf735dc0f15035813b7868e4f59179, server=27e0f852a164,41167,1733246151593 in 1.3320 sec 2024-12-03T17:17:53,284 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=107, resume processing ppid=106 2024-12-03T17:17:53,284 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, ppid=106, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=86cf735dc0f15035813b7868e4f59179, UNASSIGN in 1.3470 sec 2024-12-03T17:17:53,289 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-12-03T17:17:53,290 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.3590 sec 2024-12-03T17:17:53,292 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733246273292"}]},"ts":"1733246273292"} 2024-12-03T17:17:53,294 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-03T17:17:53,294 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-03T17:17:53,297 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.3740 sec 2024-12-03T17:17:54,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-03T17:17:54,076 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:17:54,077 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-03T17:17:54,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T17:17:54,083 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=109, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T17:17:54,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-03T17:17:54,087 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=109, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T17:17:54,106 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:54,119 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A, FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B, FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C, FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/recovered.edits] 2024-12-03T17:17:54,130 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/038c3ac7cba04c7cb87e53a8f9af9c2a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/038c3ac7cba04c7cb87e53a8f9af9c2a 2024-12-03T17:17:54,138 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/53a95a6be8154cb7b2998d6723872d49 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/A/53a95a6be8154cb7b2998d6723872d49 2024-12-03T17:17:54,149 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/2ae35bf8c6054f60859bd1a7df59c497 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/2ae35bf8c6054f60859bd1a7df59c497 2024-12-03T17:17:54,152 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/3191fac033314322bc6649406d1e314a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/B/3191fac033314322bc6649406d1e314a 2024-12-03T17:17:54,159 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/6ee16d7aa69c4a0c8c0ad7afa7041c53 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/6ee16d7aa69c4a0c8c0ad7afa7041c53 2024-12-03T17:17:54,161 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/e96eabd5b39f4e4e9cd28740506c0b2b to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/C/e96eabd5b39f4e4e9cd28740506c0b2b 2024-12-03T17:17:54,166 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/recovered.edits/462.seqid to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179/recovered.edits/462.seqid 2024-12-03T17:17:54,167 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:54,167 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-03T17:17:54,167 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-03T17:17:54,169 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-03T17:17:54,183 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203047c06021eca4582a07092b58d24b5c9_86cf735dc0f15035813b7868e4f59179 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203047c06021eca4582a07092b58d24b5c9_86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:54,191 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412030b5e5e26f6e3469887abc12f10e51d84_86cf735dc0f15035813b7868e4f59179 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412030b5e5e26f6e3469887abc12f10e51d84_86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:54,195 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203255acc78f89b466895bbfc8838489b44_86cf735dc0f15035813b7868e4f59179 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203255acc78f89b466895bbfc8838489b44_86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:54,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-03T17:17:54,200 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120337f73cc98c16409f9096a3f613651e50_86cf735dc0f15035813b7868e4f59179 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120337f73cc98c16409f9096a3f613651e50_86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:54,203 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412033c6721b432e141928484171c2466f4bc_86cf735dc0f15035813b7868e4f59179 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412033c6721b432e141928484171c2466f4bc_86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:54,205 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412037163cc217b1a4c239cd4c7b8ffacb772_86cf735dc0f15035813b7868e4f59179 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412037163cc217b1a4c239cd4c7b8ffacb772_86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:54,207 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412037cb3f3ac05e44bf4826fc5320bfee668_86cf735dc0f15035813b7868e4f59179 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412037cb3f3ac05e44bf4826fc5320bfee668_86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:54,209 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412037d50a6ad17c04620bccf72ddfb1d4dd9_86cf735dc0f15035813b7868e4f59179 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412037d50a6ad17c04620bccf72ddfb1d4dd9_86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:54,211 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039aa066998fd040ab9203ce591d0cdba8_86cf735dc0f15035813b7868e4f59179 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039aa066998fd040ab9203ce591d0cdba8_86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:54,216 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039aba100513c64982aa5e0a49dfec354f_86cf735dc0f15035813b7868e4f59179 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039aba100513c64982aa5e0a49dfec354f_86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:54,218 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039b25020819f44297bde6c506a635e58a_86cf735dc0f15035813b7868e4f59179 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039b25020819f44297bde6c506a635e58a_86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:54,222 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039c0a62c4598b4f748152ef7add32f401_86cf735dc0f15035813b7868e4f59179 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039c0a62c4598b4f748152ef7add32f401_86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:54,229 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039cd1eac19f764f6e8da2e91c7ced977c_86cf735dc0f15035813b7868e4f59179 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039cd1eac19f764f6e8da2e91c7ced977c_86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:54,230 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203a8d43ac557854440a7b155c81caa5f22_86cf735dc0f15035813b7868e4f59179 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203a8d43ac557854440a7b155c81caa5f22_86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:54,234 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203a9099f9312794effb45a02f543645fb4_86cf735dc0f15035813b7868e4f59179 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203a9099f9312794effb45a02f543645fb4_86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:54,237 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ae892f3537b340a4bd6ae5bb960fba5b_86cf735dc0f15035813b7868e4f59179 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ae892f3537b340a4bd6ae5bb960fba5b_86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:54,246 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203af32cebc89ca4b9a83e7795114d2faf3_86cf735dc0f15035813b7868e4f59179 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203af32cebc89ca4b9a83e7795114d2faf3_86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:54,249 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203bc3cefd92653446b9baeba610c2fef99_86cf735dc0f15035813b7868e4f59179 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203bc3cefd92653446b9baeba610c2fef99_86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:54,253 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203eab4be1447fc413daa9be2e4bbd37252_86cf735dc0f15035813b7868e4f59179 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203eab4be1447fc413daa9be2e4bbd37252_86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:54,263 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203fc1f05478305483aab146ff342b61d05_86cf735dc0f15035813b7868e4f59179 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203fc1f05478305483aab146ff342b61d05_86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:54,274 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203fd7a03d3b0cf4cb589f0ab5dab1b36c3_86cf735dc0f15035813b7868e4f59179 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203fd7a03d3b0cf4cb589f0ab5dab1b36c3_86cf735dc0f15035813b7868e4f59179 2024-12-03T17:17:54,279 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-03T17:17:54,283 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=109, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T17:17:54,286 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-03T17:17:54,288 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-03T17:17:54,290 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=109, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T17:17:54,290 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-03T17:17:54,290 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733246274290"}]},"ts":"9223372036854775807"} 2024-12-03T17:17:54,292 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-03T17:17:54,292 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 86cf735dc0f15035813b7868e4f59179, NAME => 'TestAcidGuarantees,,1733246240185.86cf735dc0f15035813b7868e4f59179.', STARTKEY => '', ENDKEY => ''}] 2024-12-03T17:17:54,292 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-03T17:17:54,293 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733246274292"}]},"ts":"9223372036854775807"} 2024-12-03T17:17:54,294 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-03T17:17:54,296 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=109, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T17:17:54,304 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 219 msec 2024-12-03T17:17:54,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-03T17:17:54,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-03T17:17:54,404 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:17:54,418 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=247 (was 249), OpenFileDescriptor=451 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1546 (was 1444) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4102 (was 5002) 2024-12-03T17:17:54,433 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=247, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=1546, ProcessCount=11, AvailableMemoryMB=4101 2024-12-03T17:17:54,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T17:17:54,436 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T17:17:54,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=110, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-03T17:17:54,438 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T17:17:54,439 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:17:54,440 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T17:17:54,443 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 110 2024-12-03T17:17:54,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-12-03T17:17:54,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742264_1440 (size=1061) 2024-12-03T17:17:54,503 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4 2024-12-03T17:17:54,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742265_1441 (size=53) 2024-12-03T17:17:54,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-12-03T17:17:54,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-12-03T17:17:54,934 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T17:17:54,934 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 68fbb73dc00ff4d04c231a23e30e6f1a, disabling compactions & flushes 2024-12-03T17:17:54,934 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:17:54,934 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:17:54,934 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. after waiting 0 ms 2024-12-03T17:17:54,934 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:17:54,934 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:17:54,934 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 68fbb73dc00ff4d04c231a23e30e6f1a: Waiting for close lock at 1733246274934Disabling compacts and flushes for region at 1733246274934Disabling writes for close at 1733246274934Writing region close event to WAL at 1733246274934Closed at 1733246274934 2024-12-03T17:17:54,936 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T17:17:54,936 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733246274936"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733246274936"}]},"ts":"1733246274936"} 2024-12-03T17:17:54,939 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T17:17:54,940 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T17:17:54,940 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733246274940"}]},"ts":"1733246274940"} 2024-12-03T17:17:54,942 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-03T17:17:54,943 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=68fbb73dc00ff4d04c231a23e30e6f1a, ASSIGN}] 2024-12-03T17:17:54,944 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=68fbb73dc00ff4d04c231a23e30e6f1a, ASSIGN 2024-12-03T17:17:54,945 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=68fbb73dc00ff4d04c231a23e30e6f1a, ASSIGN; state=OFFLINE, location=27e0f852a164,41167,1733246151593; forceNewPlan=false, retain=false 2024-12-03T17:17:55,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-12-03T17:17:55,096 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=68fbb73dc00ff4d04c231a23e30e6f1a, regionState=OPENING, regionLocation=27e0f852a164,41167,1733246151593 2024-12-03T17:17:55,098 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=68fbb73dc00ff4d04c231a23e30e6f1a, ASSIGN because future has completed 2024-12-03T17:17:55,098 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; OpenRegionProcedure 68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593}] 2024-12-03T17:17:55,260 INFO [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:17:55,260 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(7752): Opening region: {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} 2024-12-03T17:17:55,262 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 68fbb73dc00ff4d04c231a23e30e6f1a 2024-12-03T17:17:55,262 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T17:17:55,262 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(7794): checking encryption for 68fbb73dc00ff4d04c231a23e30e6f1a 2024-12-03T17:17:55,262 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(7797): checking classloading for 68fbb73dc00ff4d04c231a23e30e6f1a 2024-12-03T17:17:55,267 INFO [StoreOpener-68fbb73dc00ff4d04c231a23e30e6f1a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 68fbb73dc00ff4d04c231a23e30e6f1a 2024-12-03T17:17:55,269 INFO [StoreOpener-68fbb73dc00ff4d04c231a23e30e6f1a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T17:17:55,269 INFO [StoreOpener-68fbb73dc00ff4d04c231a23e30e6f1a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 68fbb73dc00ff4d04c231a23e30e6f1a columnFamilyName A 2024-12-03T17:17:55,269 DEBUG [StoreOpener-68fbb73dc00ff4d04c231a23e30e6f1a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:17:55,270 INFO [StoreOpener-68fbb73dc00ff4d04c231a23e30e6f1a-1 {}] regionserver.HStore(327): Store=68fbb73dc00ff4d04c231a23e30e6f1a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T17:17:55,270 INFO [StoreOpener-68fbb73dc00ff4d04c231a23e30e6f1a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 68fbb73dc00ff4d04c231a23e30e6f1a 2024-12-03T17:17:55,271 INFO [StoreOpener-68fbb73dc00ff4d04c231a23e30e6f1a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T17:17:55,271 INFO [StoreOpener-68fbb73dc00ff4d04c231a23e30e6f1a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 68fbb73dc00ff4d04c231a23e30e6f1a columnFamilyName B 2024-12-03T17:17:55,271 DEBUG [StoreOpener-68fbb73dc00ff4d04c231a23e30e6f1a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:17:55,272 INFO [StoreOpener-68fbb73dc00ff4d04c231a23e30e6f1a-1 {}] regionserver.HStore(327): Store=68fbb73dc00ff4d04c231a23e30e6f1a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T17:17:55,272 INFO [StoreOpener-68fbb73dc00ff4d04c231a23e30e6f1a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 68fbb73dc00ff4d04c231a23e30e6f1a 2024-12-03T17:17:55,273 INFO [StoreOpener-68fbb73dc00ff4d04c231a23e30e6f1a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T17:17:55,274 INFO [StoreOpener-68fbb73dc00ff4d04c231a23e30e6f1a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 68fbb73dc00ff4d04c231a23e30e6f1a columnFamilyName C 2024-12-03T17:17:55,274 DEBUG [StoreOpener-68fbb73dc00ff4d04c231a23e30e6f1a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:17:55,274 INFO [StoreOpener-68fbb73dc00ff4d04c231a23e30e6f1a-1 {}] regionserver.HStore(327): Store=68fbb73dc00ff4d04c231a23e30e6f1a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T17:17:55,274 INFO [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:17:55,274 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1038): replaying wal for 68fbb73dc00ff4d04c231a23e30e6f1a 2024-12-03T17:17:55,278 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a 2024-12-03T17:17:55,279 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a 2024-12-03T17:17:55,281 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1048): stopping wal replay for 68fbb73dc00ff4d04c231a23e30e6f1a 2024-12-03T17:17:55,281 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1060): Cleaning up temporary data for 68fbb73dc00ff4d04c231a23e30e6f1a 2024-12-03T17:17:55,282 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T17:17:55,283 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1093): writing seq id for 68fbb73dc00ff4d04c231a23e30e6f1a 2024-12-03T17:17:55,286 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T17:17:55,287 INFO [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1114): Opened 68fbb73dc00ff4d04c231a23e30e6f1a; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65456196, jitterRate=-0.02462667226791382}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T17:17:55,288 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 68fbb73dc00ff4d04c231a23e30e6f1a 2024-12-03T17:17:55,288 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1006): Region open journal for 68fbb73dc00ff4d04c231a23e30e6f1a: Running coprocessor pre-open hook at 1733246275262Writing region info on filesystem at 1733246275262Initializing all the Stores at 1733246275264 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733246275264Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733246275267 (+3 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733246275267Cleaning up temporary data from old regions at 1733246275281 (+14 ms)Running coprocessor post-open hooks at 1733246275288 (+7 ms)Region opened successfully at 1733246275288 2024-12-03T17:17:55,294 INFO [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., pid=112, masterSystemTime=1733246275254 2024-12-03T17:17:55,298 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=68fbb73dc00ff4d04c231a23e30e6f1a, regionState=OPEN, openSeqNum=2, regionLocation=27e0f852a164,41167,1733246151593 2024-12-03T17:17:55,298 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:17:55,298 INFO [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:17:55,301 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=112, ppid=111, state=RUNNABLE, hasLock=false; OpenRegionProcedure 68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 because future has completed 2024-12-03T17:17:55,306 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-12-03T17:17:55,306 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; OpenRegionProcedure 68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 in 205 msec 2024-12-03T17:17:55,310 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T17:17:55,310 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733246275310"}]},"ts":"1733246275310"} 2024-12-03T17:17:55,311 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=111, resume processing ppid=110 2024-12-03T17:17:55,312 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-03T17:17:55,312 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, ppid=110, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=68fbb73dc00ff4d04c231a23e30e6f1a, ASSIGN in 364 msec 2024-12-03T17:17:55,313 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T17:17:55,315 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 877 msec 2024-12-03T17:17:55,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-12-03T17:17:55,584 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:17:55,585 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@581c4bd0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:55,586 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:17:55,586 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:17:55,587 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:17:55,587 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:17:55,588 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:17:55,588 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7790b009, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:55,588 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:17:55,588 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:17:55,588 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:55,589 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54920, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:17:55,590 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dec1b63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:55,591 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:17:55,593 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:17:55,593 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:17:55,594 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48006, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:17:55,595 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T17:17:55,597 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 27e0f852a164,38523,1733246150595 2024-12-03T17:17:55,597 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@636d374e 2024-12-03T17:17:55,597 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T17:17:55,603 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54936, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T17:17:55,615 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b791feb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:55,615 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:17:55,615 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:17:55,616 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:17:55,616 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:17:55,616 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:17:55,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d46f724, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:55,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:17:55,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:17:55,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:55,618 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54952, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:17:55,619 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e28e8b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:55,631 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6da2e0be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:55,631 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:17:55,631 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:17:55,637 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:17:55,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:17:55,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:17:55,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c16de6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:55,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:17:55,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:17:55,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:55,640 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54970, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:17:55,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66eaec56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:55,645 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fc361f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:55,645 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:17:55,645 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:17:55,654 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:17:55,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:17:55,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:17:55,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ca9147, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:55,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:17:55,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:17:55,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:55,657 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54988, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:17:55,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@330eaa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:55,659 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@259a56de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:55,659 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:17:55,659 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:17:55,660 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:17:55,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:17:55,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:17:55,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a520065, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:55,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:17:55,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:17:55,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:55,662 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55002, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:17:55,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ffc6fa4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:55,663 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e4cbf6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:55,663 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:17:55,663 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:17:55,664 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:17:55,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:17:55,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:17:55,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d3cdd14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:55,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:17:55,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:17:55,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:55,665 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55016, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:17:55,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@221159d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:55,667 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b7b3b89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:55,667 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:17:55,667 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:17:55,668 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:17:55,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:17:55,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:17:55,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67ef8f09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:55,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:17:55,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:17:55,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:55,669 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55040, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:17:55,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73c6ec78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:55,673 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e04f49c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:55,673 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:17:55,673 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:17:55,674 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:17:55,674 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:17:55,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:17:55,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@576d3c9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:55,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:17:55,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:17:55,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:55,676 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55050, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:17:55,677 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7848a73f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:55,686 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32ca199c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:55,686 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:17:55,686 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:17:55,687 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:17:55,687 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:17:55,688 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:17:55,688 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bd03700, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:55,688 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:17:55,688 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:17:55,688 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:55,689 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55072, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:17:55,690 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b41fcae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:55,693 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d70caa4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:55,693 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:17:55,693 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:17:55,694 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:17:55,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:17:55,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:17:55,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40dd3a8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:55,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:17:55,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:17:55,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:55,696 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55092, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:17:55,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6393cf6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:55,698 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43602cdd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:55,698 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:17:55,698 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:17:55,699 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:17:55,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:17:55,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:17:55,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32f52857, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:55,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:17:55,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:17:55,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:17:55,700 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55114, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:17:55,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d5e7ddf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:17:55,703 DEBUG [Thread-1990 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:17:55,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:17:55,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:17:55,705 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48016, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:17:55,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2] 2024-12-03T17:17:55,711 DEBUG [Thread-1993 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:17:55,711 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:17:55,712 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:17:55,713 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48018, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:17:55,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2] 2024-12-03T17:17:55,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 68fbb73dc00ff4d04c231a23e30e6f1a 2024-12-03T17:17:55,722 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 68fbb73dc00ff4d04c231a23e30e6f1a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T17:17:55,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=A 2024-12-03T17:17:55,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:55,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=B 2024-12-03T17:17:55,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:55,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=C 2024-12-03T17:17:55,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:55,727 DEBUG [Thread-1988 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:17:55,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:17:55,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:17:55,730 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48024, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:17:55,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2] 2024-12-03T17:17:55,751 DEBUG [Thread-1997 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:17:55,752 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:17:55,752 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:17:55,753 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48040, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:17:55,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2] 2024-12-03T17:17:55,766 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/afe4274cd3d3408198dbc2c6a7d06367 is 50, key is test_row_0/A:col10/1733246275719/Put/seqid=0 2024-12-03T17:17:55,767 DEBUG [Thread-1984 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:17:55,770 DEBUG [Thread-1984 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:17:55,771 DEBUG [Thread-1984 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:17:55,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733246275746, max timestamp=1733246275758Num uniques 859; 2024-12-03T17:17:55,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733246275746, max timestamp=1733246275758Num uniques 859; 2024-12-03T17:17:55,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T17:17:55,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T17:17:55,775 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48042, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:17:55,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2] 2024-12-03T17:17:55,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733246275746, max timestamp=1733246275758Num uniques 859; 2024-12-03T17:17:55,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T17:17:55,786 DEBUG [Thread-1986 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:17:55,788 DEBUG [Thread-1999 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:17:55,789 DEBUG [Thread-1999 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:17:55,789 DEBUG [Thread-1999 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:17:55,790 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48048, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:17:55,791 DEBUG [Thread-1995 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:17:55,794 DEBUG [Thread-1986 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:17:55,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2] 2024-12-03T17:17:55,799 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:17:55,799 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:17:55,803 DEBUG [Thread-1986 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:17:55,803 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48054, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:17:55,806 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48068, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:17:55,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2] 2024-12-03T17:17:55,812 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2] 2024-12-03T17:17:55,814 DEBUG [Thread-1982 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:17:55,817 DEBUG [Thread-1982 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:17:55,817 DEBUG [Thread-1982 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:17:55,820 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48080, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:17:55,821 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2] 2024-12-03T17:17:55,827 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T17:17:55,835 DEBUG [Thread-2001 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:17:55,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:17:55,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:17:55,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-12-03T17:17:55,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-03T17:17:55,841 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48086, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:17:55,856 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T17:17:55,857 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T17:17:55,857 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T17:17:55,857 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2] 2024-12-03T17:17:55,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:55,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48068 deadline: 1733246335845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:55,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:55,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246335854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:55,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:55,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:55,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:55,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:55,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246335855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:55,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:55,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:55,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:55,899 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:55,899 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:55,899 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:55,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742266_1442 (size=14337) 2024-12-03T17:17:55,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:55,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246335870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:55,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:55,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:55,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:55,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:55,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246335871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:55,922 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:55,922 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:55,922 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:55,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-03T17:17:56,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-03T17:17:56,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:17:56,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. as already flushing 2024-12-03T17:17:56,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:17:56,021 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:56,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:56,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:56,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:56,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246336005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:56,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:56,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246336005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:56,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:56,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48068 deadline: 1733246336005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:56,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:56,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:56,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:56,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:56,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:56,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:56,039 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:56,039 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:56,039 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:56,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:56,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246336027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:56,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:56,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:56,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:56,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:56,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246336027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:56,041 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:56,041 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:56,041 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:56,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-03T17:17:56,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-03T17:17:56,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:17:56,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. as already flushing 2024-12-03T17:17:56,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:17:56,183 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:56,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:56,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:56,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:56,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246336244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:56,248 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:56,248 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:56,248 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:56,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:56,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246336244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:56,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:56,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:56,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:56,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:56,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48068 deadline: 1733246336246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:56,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:56,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246336247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:56,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:56,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:56,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:56,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:56,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246336247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:56,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:56,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:56,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:56,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:56,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:56,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:56,310 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/afe4274cd3d3408198dbc2c6a7d06367 2024-12-03T17:17:56,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-03T17:17:56,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:17:56,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. as already flushing 2024-12-03T17:17:56,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:17:56,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:56,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:56,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:56,407 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/6404fdd485dd48d5998c5fa6387618d5 is 50, key is test_row_0/B:col10/1733246275719/Put/seqid=0 2024-12-03T17:17:56,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742267_1443 (size=9657) 2024-12-03T17:17:56,459 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/6404fdd485dd48d5998c5fa6387618d5 2024-12-03T17:17:56,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-03T17:17:56,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/a46e339b04fa4b0d833d529e1f99c8a9 is 50, key is test_row_0/C:col10/1733246275719/Put/seqid=0 2024-12-03T17:17:56,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742268_1444 (size=9657) 2024-12-03T17:17:56,554 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-03T17:17:56,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:17:56,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. as already flushing 2024-12-03T17:17:56,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:17:56,555 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:56,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:56,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:56,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:56,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246336554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:56,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:56,571 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:56,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246336554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:56,571 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:56,571 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:56,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:56,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:56,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:56,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:56,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48068 deadline: 1733246336574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:56,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:56,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:56,590 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:56,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:56,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246336574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:56,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:56,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:56,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:56,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:56,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246336575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:56,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:56,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:56,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:56,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-03T17:17:56,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:17:56,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. as already flushing 2024-12-03T17:17:56,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:17:56,709 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:56,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:56,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:56,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-03T17:17:56,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:17:56,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. as already flushing 2024-12-03T17:17:56,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:17:56,862 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:56,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:56,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:56,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/a46e339b04fa4b0d833d529e1f99c8a9 2024-12-03T17:17:56,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-03T17:17:56,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/afe4274cd3d3408198dbc2c6a7d06367 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/afe4274cd3d3408198dbc2c6a7d06367 2024-12-03T17:17:57,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-03T17:17:57,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:17:57,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. as already flushing 2024-12-03T17:17:57,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:17:57,018 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:57,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:57,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:57,023 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/afe4274cd3d3408198dbc2c6a7d06367, entries=200, sequenceid=11, filesize=14.0 K 2024-12-03T17:17:57,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/6404fdd485dd48d5998c5fa6387618d5 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/6404fdd485dd48d5998c5fa6387618d5 2024-12-03T17:17:57,029 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/6404fdd485dd48d5998c5fa6387618d5, entries=100, sequenceid=11, filesize=9.4 K 2024-12-03T17:17:57,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/a46e339b04fa4b0d833d529e1f99c8a9 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/a46e339b04fa4b0d833d529e1f99c8a9 2024-12-03T17:17:57,034 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/a46e339b04fa4b0d833d529e1f99c8a9, entries=100, sequenceid=11, filesize=9.4 K 2024-12-03T17:17:57,035 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 68fbb73dc00ff4d04c231a23e30e6f1a in 1313ms, sequenceid=11, compaction requested=false 2024-12-03T17:17:57,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 68fbb73dc00ff4d04c231a23e30e6f1a: 2024-12-03T17:17:57,089 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733246275835, max timestamp=1733246277084Num uniques 1068; 2024-12-03T17:17:57,089 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-03T17:17:57,094 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733246275835, max timestamp=1733246277084Num uniques 1068; 2024-12-03T17:17:57,094 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733246275835, max timestamp=1733246277084Num uniques 1068; 2024-12-03T17:17:57,094 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-03T17:17:57,094 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-03T17:17:57,110 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 68fbb73dc00ff4d04c231a23e30e6f1a 3/3 column families, dataSize=53.67 KB heapSize=145.08 KB 2024-12-03T17:17:57,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=A 2024-12-03T17:17:57,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:17:57,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=B 2024-12-03T17:17:57,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:17:57,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=C 2024-12-03T17:17:57,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:17:57,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 68fbb73dc00ff4d04c231a23e30e6f1a 2024-12-03T17:17:57,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/b61da39d6b9b4196bdddfc3105a6ddef is 50, key is test_row_0/A:col10/1733246277109/Put/seqid=0 2024-12-03T17:17:57,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-03T17:17:57,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:17:57,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. as already flushing 2024-12-03T17:17:57,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:17:57,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:57,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:57,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:57,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742269_1445 (size=14341) 2024-12-03T17:17:57,235 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733246277175, max timestamp=1733246277232Num uniques 859; 2024-12-03T17:17:57,235 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T17:17:57,237 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733246277175, max timestamp=1733246277232Num uniques 859; 2024-12-03T17:17:57,237 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T17:17:57,240 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733246277175, max timestamp=1733246277232Num uniques 859; 2024-12-03T17:17:57,240 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T17:17:57,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-03T17:17:57,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:17:57,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. as already flushing 2024-12-03T17:17:57,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:17:57,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:57,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:57,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:57,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:57,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48068 deadline: 1733246337355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:57,396 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:57,396 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:57,396 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:57,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:57,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246337385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:57,421 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:57,421 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:57,421 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:57,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:57,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246337387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:57,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:57,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246337392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:57,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:57,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:57,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:57,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:57,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:57,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:57,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:57,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246337396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:57,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:57,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:57,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:57,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-03T17:17:57,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:17:57,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. as already flushing 2024-12-03T17:17:57,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:17:57,495 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:57,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:57,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:57,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:57,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48068 deadline: 1733246337511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:57,537 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:57,537 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:57,537 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:57,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:57,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246337523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:57,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:57,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:57,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:57,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:57,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246337534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:57,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:57,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:57,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:57,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246337534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:57,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:57,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:57,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:57,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:57,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:57,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246337534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:57,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:57,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:57,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:57,622 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/b61da39d6b9b4196bdddfc3105a6ddef 2024-12-03T17:17:57,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/dac2fef6753e45d787fc73ba428c58e8 is 50, key is test_row_0/B:col10/1733246277109/Put/seqid=0 2024-12-03T17:17:57,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-03T17:17:57,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:17:57,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. as already flushing 2024-12-03T17:17:57,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:17:57,654 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:57,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:57,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:57,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742270_1446 (size=12001) 2024-12-03T17:17:57,707 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/dac2fef6753e45d787fc73ba428c58e8 2024-12-03T17:17:57,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/1a5aa931bf6048129e090251e04f84d4 is 50, key is test_row_0/C:col10/1733246277109/Put/seqid=0 2024-12-03T17:17:57,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742271_1447 (size=12001) 2024-12-03T17:17:57,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:57,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48068 deadline: 1733246337747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:57,771 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:57,771 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:57,771 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:57,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:57,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246337754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:57,772 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:57,772 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:57,772 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:57,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:57,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246337773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:57,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:57,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:57,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:57,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:57,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246337774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:57,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:57,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:57,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246337774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:57,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:57,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:57,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:57,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:57,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:57,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-03T17:17:57,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:17:57,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. as already flushing 2024-12-03T17:17:57,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:17:57,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:57,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:57,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:57,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-03T17:17:57,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:17:57,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. as already flushing 2024-12-03T17:17:57,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:17:57,974 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:57,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:57,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:57,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-03T17:17:58,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:58,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48068 deadline: 1733246338074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:58,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:58,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:58,085 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:58,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:58,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246338085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:58,109 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:58,109 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:58,109 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:58,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:58,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246338103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:58,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-03T17:17:58,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:58,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:58,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:17:58,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246338104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:58,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246338104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:58,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. as already flushing 2024-12-03T17:17:58,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:17:58,132 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:58,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:58,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:58,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:58,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:58,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:58,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:58,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:58,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:17:58,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:58,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:58,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:58,166 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/1a5aa931bf6048129e090251e04f84d4 2024-12-03T17:17:58,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/b61da39d6b9b4196bdddfc3105a6ddef as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/b61da39d6b9b4196bdddfc3105a6ddef 2024-12-03T17:17:58,187 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/b61da39d6b9b4196bdddfc3105a6ddef, entries=200, sequenceid=53, filesize=14.0 K 2024-12-03T17:17:58,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/dac2fef6753e45d787fc73ba428c58e8 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/dac2fef6753e45d787fc73ba428c58e8 2024-12-03T17:17:58,201 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/dac2fef6753e45d787fc73ba428c58e8, entries=150, sequenceid=53, filesize=11.7 K 2024-12-03T17:17:58,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/1a5aa931bf6048129e090251e04f84d4 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/1a5aa931bf6048129e090251e04f84d4 2024-12-03T17:17:58,208 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/1a5aa931bf6048129e090251e04f84d4, entries=150, sequenceid=53, filesize=11.7 K 2024-12-03T17:17:58,210 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.03 KB/148512, currentSize=134.18 KB/137400 for 68fbb73dc00ff4d04c231a23e30e6f1a in 1100ms, sequenceid=53, compaction requested=false 2024-12-03T17:17:58,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 68fbb73dc00ff4d04c231a23e30e6f1a: 2024-12-03T17:17:58,292 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-03T17:17:58,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:17:58,292 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 68fbb73dc00ff4d04c231a23e30e6f1a 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-03T17:17:58,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=A 2024-12-03T17:17:58,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:17:58,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=B 2024-12-03T17:17:58,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:17:58,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=C 2024-12-03T17:17:58,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:17:58,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/ad15a0e79f6d4587a564a22114d76754 is 50, key is test_row_0/A:col10/1733246277390/Put/seqid=0 2024-12-03T17:17:58,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742272_1448 (size=12001) 2024-12-03T17:17:58,363 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/ad15a0e79f6d4587a564a22114d76754 2024-12-03T17:17:58,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/51d6cb44a94e48d9a57bf6099b4d800f is 50, key is test_row_0/B:col10/1733246277390/Put/seqid=0 2024-12-03T17:17:58,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742273_1449 (size=12001) 2024-12-03T17:17:58,463 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/51d6cb44a94e48d9a57bf6099b4d800f 2024-12-03T17:17:58,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/d2463cb5e46f44de922761ea78fdf381 is 50, key is test_row_0/C:col10/1733246277390/Put/seqid=0 2024-12-03T17:17:58,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742274_1450 (size=12001) 2024-12-03T17:17:58,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 68fbb73dc00ff4d04c231a23e30e6f1a 2024-12-03T17:17:58,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. as already flushing 2024-12-03T17:17:58,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:58,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246338691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:58,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:58,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:58,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:58,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:58,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:58,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48068 deadline: 1733246338697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:58,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246338691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:58,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:58,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246338692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:58,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:58,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:58,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:58,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:58,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246338702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:58,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:58,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:58,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:58,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:58,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:58,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:58,736 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:58,736 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:58,736 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:58,775 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T17:17:58,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:58,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246338825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:58,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:58,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:58,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:58,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:58,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48068 deadline: 1733246338844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:58,860 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:58,860 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:58,860 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:58,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:58,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246338844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:58,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:58,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:58,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:58,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:58,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246338845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:58,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:58,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:58,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:58,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:58,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246338845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:58,862 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:58,862 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:58,862 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:58,951 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/d2463cb5e46f44de922761ea78fdf381 2024-12-03T17:17:58,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/ad15a0e79f6d4587a564a22114d76754 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/ad15a0e79f6d4587a564a22114d76754 2024-12-03T17:17:58,964 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/ad15a0e79f6d4587a564a22114d76754, entries=150, sequenceid=90, filesize=11.7 K 2024-12-03T17:17:58,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/51d6cb44a94e48d9a57bf6099b4d800f as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/51d6cb44a94e48d9a57bf6099b4d800f 2024-12-03T17:17:58,978 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/51d6cb44a94e48d9a57bf6099b4d800f, entries=150, sequenceid=90, filesize=11.7 K 2024-12-03T17:17:58,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/d2463cb5e46f44de922761ea78fdf381 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/d2463cb5e46f44de922761ea78fdf381 2024-12-03T17:17:58,992 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/d2463cb5e46f44de922761ea78fdf381, entries=150, sequenceid=90, filesize=11.7 K 2024-12-03T17:17:58,995 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 68fbb73dc00ff4d04c231a23e30e6f1a in 703ms, sequenceid=90, compaction requested=true 2024-12-03T17:17:58,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 68fbb73dc00ff4d04c231a23e30e6f1a: 2024-12-03T17:17:58,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:17:58,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-12-03T17:17:59,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-12-03T17:17:59,007 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-12-03T17:17:59,007 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1480 sec 2024-12-03T17:17:59,009 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 3.1810 sec 2024-12-03T17:17:59,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 68fbb73dc00ff4d04c231a23e30e6f1a 2024-12-03T17:17:59,060 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 68fbb73dc00ff4d04c231a23e30e6f1a 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-03T17:17:59,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=A 2024-12-03T17:17:59,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:59,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=B 2024-12-03T17:17:59,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:59,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=C 2024-12-03T17:17:59,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:17:59,084 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/8b1cb59d5fca4511b900ba8a2a176a36 is 50, key is test_row_0/A:col10/1733246278695/Put/seqid=0 2024-12-03T17:17:59,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742275_1451 (size=14341) 2024-12-03T17:17:59,139 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/8b1cb59d5fca4511b900ba8a2a176a36 2024-12-03T17:17:59,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/c7529491e4934f428d46fe1ffcaa1b40 is 50, key is test_row_0/B:col10/1733246278695/Put/seqid=0 2024-12-03T17:17:59,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:59,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246339146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:59,175 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:59,175 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:59,175 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:59,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:59,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246339148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:59,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:59,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:59,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:59,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:59,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246339150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:59,177 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733246279108, max timestamp=1733246279169Num uniques 859; 2024-12-03T17:17:59,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:59,177 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T17:17:59,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:59,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:59,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733246279108, max timestamp=1733246279169Num uniques 859; 2024-12-03T17:17:59,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T17:17:59,177 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733246279108, max timestamp=1733246279169Num uniques 859; 2024-12-03T17:17:59,182 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T17:17:59,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742276_1452 (size=12001) 2024-12-03T17:17:59,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:59,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246339348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:59,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:59,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:59,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246339349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:59,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:59,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:59,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:59,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:59,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:59,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:59,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48068 deadline: 1733246339389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:59,433 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:59,433 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:59,433 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:59,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:59,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246339391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:59,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:59,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:59,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:59,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:59,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246339393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:59,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:59,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:59,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:59,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:59,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246339504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:59,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:59,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:59,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:59,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:59,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246339504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:59,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:59,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:59,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:59,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:59,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246339544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:59,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:59,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246339544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:59,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:59,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:59,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:59,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:59,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48068 deadline: 1733246339544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:59,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:59,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:59,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:59,560 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:59,560 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:59,560 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:59,599 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/c7529491e4934f428d46fe1ffcaa1b40 2024-12-03T17:17:59,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/eddf120eefb5498d92801fc5ced81a96 is 50, key is test_row_0/C:col10/1733246278695/Put/seqid=0 2024-12-03T17:17:59,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742277_1453 (size=12001) 2024-12-03T17:17:59,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:59,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246339724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:59,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:59,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246339724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:59,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:59,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:59,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:59,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:59,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:59,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:59,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:59,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246339764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:59,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:59,773 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:59,773 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:59,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48068 deadline: 1733246339764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:59,773 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:59,773 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:59,773 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:59,773 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:17:59,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:17:59,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246339764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:17:59,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:59,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:17:59,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:00,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-03T17:18:00,006 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:18:00,010 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T17:18:00,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-12-03T17:18:00,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-03T17:18:00,021 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T17:18:00,022 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T17:18:00,022 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T17:18:00,057 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/eddf120eefb5498d92801fc5ced81a96 2024-12-03T17:18:00,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/8b1cb59d5fca4511b900ba8a2a176a36 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/8b1cb59d5fca4511b900ba8a2a176a36 2024-12-03T17:18:00,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:00,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246340056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:00,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:00,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:00,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:00,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:00,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246340057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:00,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:00,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:00,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:00,092 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/8b1cb59d5fca4511b900ba8a2a176a36, entries=200, sequenceid=101, filesize=14.0 K 2024-12-03T17:18:00,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/c7529491e4934f428d46fe1ffcaa1b40 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/c7529491e4934f428d46fe1ffcaa1b40 2024-12-03T17:18:00,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:00,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48068 deadline: 1733246340084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:00,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:00,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246340084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:00,096 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:00,096 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:00,096 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:00,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:00,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:00,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:00,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:00,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246340084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:00,102 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:00,102 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:00,102 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:00,110 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/c7529491e4934f428d46fe1ffcaa1b40, entries=150, sequenceid=101, filesize=11.7 K 2024-12-03T17:18:00,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/eddf120eefb5498d92801fc5ced81a96 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/eddf120eefb5498d92801fc5ced81a96 2024-12-03T17:18:00,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-03T17:18:00,128 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/eddf120eefb5498d92801fc5ced81a96, entries=150, sequenceid=101, filesize=11.7 K 2024-12-03T17:18:00,132 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 68fbb73dc00ff4d04c231a23e30e6f1a in 1072ms, sequenceid=101, compaction requested=true 2024-12-03T17:18:00,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 68fbb73dc00ff4d04c231a23e30e6f1a: 2024-12-03T17:18:00,133 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T17:18:00,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68fbb73dc00ff4d04c231a23e30e6f1a:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T17:18:00,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:18:00,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68fbb73dc00ff4d04c231a23e30e6f1a:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T17:18:00,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:18:00,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68fbb73dc00ff4d04c231a23e30e6f1a:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T17:18:00,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:18:00,133 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T17:18:00,138 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55020 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T17:18:00,139 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): 68fbb73dc00ff4d04c231a23e30e6f1a/A is initiating minor compaction (all files) 2024-12-03T17:18:00,139 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68fbb73dc00ff4d04c231a23e30e6f1a/A in TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:00,139 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/afe4274cd3d3408198dbc2c6a7d06367, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/b61da39d6b9b4196bdddfc3105a6ddef, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/ad15a0e79f6d4587a564a22114d76754, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/8b1cb59d5fca4511b900ba8a2a176a36] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp, totalSize=53.7 K 2024-12-03T17:18:00,139 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T17:18:00,139 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): 68fbb73dc00ff4d04c231a23e30e6f1a/B is initiating minor compaction (all files) 2024-12-03T17:18:00,139 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68fbb73dc00ff4d04c231a23e30e6f1a/B in TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:00,140 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/6404fdd485dd48d5998c5fa6387618d5, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/dac2fef6753e45d787fc73ba428c58e8, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/51d6cb44a94e48d9a57bf6099b4d800f, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/c7529491e4934f428d46fe1ffcaa1b40] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp, totalSize=44.6 K 2024-12-03T17:18:00,140 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting afe4274cd3d3408198dbc2c6a7d06367, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733246275711 2024-12-03T17:18:00,140 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6404fdd485dd48d5998c5fa6387618d5, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733246275719 2024-12-03T17:18:00,140 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting b61da39d6b9b4196bdddfc3105a6ddef, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1733246277084 2024-12-03T17:18:00,141 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting dac2fef6753e45d787fc73ba428c58e8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1733246277103 2024-12-03T17:18:00,141 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting ad15a0e79f6d4587a564a22114d76754, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1733246277379 2024-12-03T17:18:00,141 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51d6cb44a94e48d9a57bf6099b4d800f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1733246277379 2024-12-03T17:18:00,141 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b1cb59d5fca4511b900ba8a2a176a36, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733246278686 2024-12-03T17:18:00,142 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7529491e4934f428d46fe1ffcaa1b40, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733246278686 2024-12-03T17:18:00,173 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68fbb73dc00ff4d04c231a23e30e6f1a#B#compaction#378 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:18:00,174 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/54ad9b0c5b2c46aab7a67c87671a4b00 is 50, key is test_row_0/B:col10/1733246278695/Put/seqid=0 2024-12-03T17:18:00,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-03T17:18:00,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:00,178 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 68fbb73dc00ff4d04c231a23e30e6f1a 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-12-03T17:18:00,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=A 2024-12-03T17:18:00,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:18:00,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=B 2024-12-03T17:18:00,183 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68fbb73dc00ff4d04c231a23e30e6f1a#A#compaction#379 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:18:00,183 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/a43860ea6f0b4aedae8081309a099397 is 50, key is test_row_0/A:col10/1733246278695/Put/seqid=0 2024-12-03T17:18:00,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:18:00,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=C 2024-12-03T17:18:00,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:18:00,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742278_1454 (size=12139) 2024-12-03T17:18:00,264 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/54ad9b0c5b2c46aab7a67c87671a4b00 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/54ad9b0c5b2c46aab7a67c87671a4b00 2024-12-03T17:18:00,270 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 68fbb73dc00ff4d04c231a23e30e6f1a/B of 68fbb73dc00ff4d04c231a23e30e6f1a into 54ad9b0c5b2c46aab7a67c87671a4b00(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:18:00,270 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68fbb73dc00ff4d04c231a23e30e6f1a: 2024-12-03T17:18:00,270 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., storeName=68fbb73dc00ff4d04c231a23e30e6f1a/B, priority=12, startTime=1733246280133; duration=0sec 2024-12-03T17:18:00,270 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:18:00,270 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68fbb73dc00ff4d04c231a23e30e6f1a:B 2024-12-03T17:18:00,270 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T17:18:00,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/2dc30052fa0f447f9176bf4bf84b502e is 50, key is test_row_0/A:col10/1733246279386/Put/seqid=0 2024-12-03T17:18:00,276 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T17:18:00,276 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): 68fbb73dc00ff4d04c231a23e30e6f1a/C is initiating minor compaction (all files) 2024-12-03T17:18:00,276 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68fbb73dc00ff4d04c231a23e30e6f1a/C in TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:00,277 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/a46e339b04fa4b0d833d529e1f99c8a9, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/1a5aa931bf6048129e090251e04f84d4, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/d2463cb5e46f44de922761ea78fdf381, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/eddf120eefb5498d92801fc5ced81a96] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp, totalSize=44.6 K 2024-12-03T17:18:00,277 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting a46e339b04fa4b0d833d529e1f99c8a9, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733246275719 2024-12-03T17:18:00,277 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a5aa931bf6048129e090251e04f84d4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1733246277103 2024-12-03T17:18:00,278 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2463cb5e46f44de922761ea78fdf381, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1733246277379 2024-12-03T17:18:00,278 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting eddf120eefb5498d92801fc5ced81a96, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733246278686 2024-12-03T17:18:00,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742279_1455 (size=12139) 2024-12-03T17:18:00,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-03T17:18:00,338 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68fbb73dc00ff4d04c231a23e30e6f1a#C#compaction#381 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:18:00,339 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/64780699e14e41e7878f63426c25acba is 50, key is test_row_0/C:col10/1733246278695/Put/seqid=0 2024-12-03T17:18:00,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742280_1456 (size=12151) 2024-12-03T17:18:00,359 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/2dc30052fa0f447f9176bf4bf84b502e 2024-12-03T17:18:00,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742281_1457 (size=12139) 2024-12-03T17:18:00,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/a069caafc2ea4f6ebfe3ba6711a1d6af is 50, key is test_row_0/B:col10/1733246279386/Put/seqid=0 2024-12-03T17:18:00,385 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/64780699e14e41e7878f63426c25acba as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/64780699e14e41e7878f63426c25acba 2024-12-03T17:18:00,394 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 68fbb73dc00ff4d04c231a23e30e6f1a/C of 68fbb73dc00ff4d04c231a23e30e6f1a into 64780699e14e41e7878f63426c25acba(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:18:00,394 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68fbb73dc00ff4d04c231a23e30e6f1a: 2024-12-03T17:18:00,394 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., storeName=68fbb73dc00ff4d04c231a23e30e6f1a/C, priority=12, startTime=1733246280133; duration=0sec 2024-12-03T17:18:00,395 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:18:00,395 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68fbb73dc00ff4d04c231a23e30e6f1a:C 2024-12-03T17:18:00,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742282_1458 (size=12151) 2024-12-03T17:18:00,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. as already flushing 2024-12-03T17:18:00,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 68fbb73dc00ff4d04c231a23e30e6f1a 2024-12-03T17:18:00,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:00,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246340619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:00,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:00,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:00,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:00,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-03T17:18:00,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:00,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246340619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:00,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:00,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48068 deadline: 1733246340627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:00,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:00,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:00,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:00,649 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:00,649 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:00,649 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:00,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:00,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246340639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:00,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:00,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:00,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:00,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:00,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246340646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:00,670 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:00,670 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:00,671 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:00,720 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/a43860ea6f0b4aedae8081309a099397 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/a43860ea6f0b4aedae8081309a099397 2024-12-03T17:18:00,733 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 68fbb73dc00ff4d04c231a23e30e6f1a/A of 68fbb73dc00ff4d04c231a23e30e6f1a into a43860ea6f0b4aedae8081309a099397(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:18:00,733 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68fbb73dc00ff4d04c231a23e30e6f1a: 2024-12-03T17:18:00,733 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., storeName=68fbb73dc00ff4d04c231a23e30e6f1a/A, priority=12, startTime=1733246280133; duration=0sec 2024-12-03T17:18:00,733 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:18:00,733 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68fbb73dc00ff4d04c231a23e30e6f1a:A 2024-12-03T17:18:00,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:00,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246340753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:00,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:00,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:00,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246340753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:00,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:00,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:00,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:00,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48068 deadline: 1733246340753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:00,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:00,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:00,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:00,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:00,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:00,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:00,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:00,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246340774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:00,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:00,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:00,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246340774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:00,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:00,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:00,785 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:00,785 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:00,785 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:00,811 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/a069caafc2ea4f6ebfe3ba6711a1d6af 2024-12-03T17:18:00,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/c6f754999a1f4da0ad194e058cb0953a is 50, key is test_row_0/C:col10/1733246279386/Put/seqid=0 2024-12-03T17:18:00,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742283_1459 (size=12151) 2024-12-03T17:18:00,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:00,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246340974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:00,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:00,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246340974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:00,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:00,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48068 deadline: 1733246340974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:00,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:00,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:00,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:00,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:00,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:00,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:00,986 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:00,986 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:00,986 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:00,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:01,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246340997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:01,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:01,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246340997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:01,000 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:01,000 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:01,001 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:01,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:01,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:01,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:01,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-03T17:18:01,253 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/c6f754999a1f4da0ad194e058cb0953a 2024-12-03T17:18:01,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/2dc30052fa0f447f9176bf4bf84b502e as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/2dc30052fa0f447f9176bf4bf84b502e 2024-12-03T17:18:01,288 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/2dc30052fa0f447f9176bf4bf84b502e, entries=150, sequenceid=136, filesize=11.9 K 2024-12-03T17:18:01,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/a069caafc2ea4f6ebfe3ba6711a1d6af as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/a069caafc2ea4f6ebfe3ba6711a1d6af 2024-12-03T17:18:01,300 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/a069caafc2ea4f6ebfe3ba6711a1d6af, entries=150, sequenceid=136, filesize=11.9 K 2024-12-03T17:18:01,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/c6f754999a1f4da0ad194e058cb0953a as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/c6f754999a1f4da0ad194e058cb0953a 2024-12-03T17:18:01,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:01,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246341293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:01,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:01,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:01,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246341294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:01,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:01,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:01,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:01,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48068 deadline: 1733246341294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:01,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:01,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:01,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:01,313 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:01,313 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:01,313 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:01,315 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/c6f754999a1f4da0ad194e058cb0953a, entries=150, sequenceid=136, filesize=11.9 K 2024-12-03T17:18:01,317 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 68fbb73dc00ff4d04c231a23e30e6f1a in 1139ms, sequenceid=136, compaction requested=false 2024-12-03T17:18:01,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 68fbb73dc00ff4d04c231a23e30e6f1a: 2024-12-03T17:18:01,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:01,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-12-03T17:18:01,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-12-03T17:18:01,327 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 68fbb73dc00ff4d04c231a23e30e6f1a 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T17:18:01,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=A 2024-12-03T17:18:01,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:01,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=B 2024-12-03T17:18:01,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:01,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=C 2024-12-03T17:18:01,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:01,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 68fbb73dc00ff4d04c231a23e30e6f1a 2024-12-03T17:18:01,329 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-12-03T17:18:01,329 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2980 sec 2024-12-03T17:18:01,333 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.3200 sec 2024-12-03T17:18:01,333 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/4dec81aff97f4669ad73b4b0619c7a0b is 50, key is test_row_0/A:col10/1733246281322/Put/seqid=0 2024-12-03T17:18:01,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742284_1460 (size=16931) 2024-12-03T17:18:01,377 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/4dec81aff97f4669ad73b4b0619c7a0b 2024-12-03T17:18:01,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/e4bdb5853cbb496c9152fda9a0ac0896 is 50, key is test_row_0/B:col10/1733246281322/Put/seqid=0 2024-12-03T17:18:01,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742285_1461 (size=12151) 2024-12-03T17:18:01,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:01,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246341519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:01,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:01,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:01,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246341523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:01,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:01,541 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:01,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:01,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:01,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:01,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:01,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246341644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:01,654 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:01,654 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:01,654 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:01,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:01,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246341657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:01,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:01,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:01,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:01,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:01,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246341825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:01,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:01,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:01,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:01,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:01,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246341826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:01,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:01,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:01,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:01,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:01,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48068 deadline: 1733246341827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:01,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:01,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:01,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:01,850 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/e4bdb5853cbb496c9152fda9a0ac0896 2024-12-03T17:18:01,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:01,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246341868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:01,870 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:01,870 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:01,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:01,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/2f3b1bdea074418680fc54179ecf26e1 is 50, key is test_row_0/C:col10/1733246281322/Put/seqid=0 2024-12-03T17:18:01,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:01,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246341889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:01,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:01,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:01,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:01,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742286_1462 (size=12151) 2024-12-03T17:18:01,914 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/2f3b1bdea074418680fc54179ecf26e1 2024-12-03T17:18:01,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/4dec81aff97f4669ad73b4b0619c7a0b as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/4dec81aff97f4669ad73b4b0619c7a0b 2024-12-03T17:18:01,924 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/4dec81aff97f4669ad73b4b0619c7a0b, entries=250, sequenceid=152, filesize=16.5 K 2024-12-03T17:18:01,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/e4bdb5853cbb496c9152fda9a0ac0896 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/e4bdb5853cbb496c9152fda9a0ac0896 2024-12-03T17:18:01,929 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/e4bdb5853cbb496c9152fda9a0ac0896, entries=150, sequenceid=152, filesize=11.9 K 2024-12-03T17:18:01,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/2f3b1bdea074418680fc54179ecf26e1 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/2f3b1bdea074418680fc54179ecf26e1 2024-12-03T17:18:01,934 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/2f3b1bdea074418680fc54179ecf26e1, entries=150, sequenceid=152, filesize=11.9 K 2024-12-03T17:18:01,943 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 68fbb73dc00ff4d04c231a23e30e6f1a in 617ms, sequenceid=152, compaction requested=true 2024-12-03T17:18:01,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 68fbb73dc00ff4d04c231a23e30e6f1a: 2024-12-03T17:18:01,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68fbb73dc00ff4d04c231a23e30e6f1a:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T17:18:01,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:18:01,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68fbb73dc00ff4d04c231a23e30e6f1a:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T17:18:01,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T17:18:01,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68fbb73dc00ff4d04c231a23e30e6f1a:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T17:18:01,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-03T17:18:01,943 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:18:01,944 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41221 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:18:01,944 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): 68fbb73dc00ff4d04c231a23e30e6f1a/A is initiating minor compaction (all files) 2024-12-03T17:18:01,944 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68fbb73dc00ff4d04c231a23e30e6f1a/A in TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:01,945 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/a43860ea6f0b4aedae8081309a099397, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/2dc30052fa0f447f9176bf4bf84b502e, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/4dec81aff97f4669ad73b4b0619c7a0b] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp, totalSize=40.3 K 2024-12-03T17:18:01,945 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting a43860ea6f0b4aedae8081309a099397, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733246278686 2024-12-03T17:18:01,945 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 2dc30052fa0f447f9176bf4bf84b502e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733246279386 2024-12-03T17:18:01,946 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 4dec81aff97f4669ad73b4b0619c7a0b, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733246280609 2024-12-03T17:18:01,949 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:18:01,958 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36441 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:18:01,958 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): 68fbb73dc00ff4d04c231a23e30e6f1a/C is initiating minor compaction (all files) 2024-12-03T17:18:01,959 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68fbb73dc00ff4d04c231a23e30e6f1a/C in TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:01,959 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/64780699e14e41e7878f63426c25acba, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/c6f754999a1f4da0ad194e058cb0953a, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/2f3b1bdea074418680fc54179ecf26e1] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp, totalSize=35.6 K 2024-12-03T17:18:01,963 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64780699e14e41e7878f63426c25acba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733246278686 2024-12-03T17:18:01,964 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6f754999a1f4da0ad194e058cb0953a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733246279386 2024-12-03T17:18:01,964 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f3b1bdea074418680fc54179ecf26e1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733246280611 2024-12-03T17:18:01,975 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68fbb73dc00ff4d04c231a23e30e6f1a#A#compaction#387 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:18:01,975 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/03c9ddf2ef0c461288201c51a63206a3 is 50, key is test_row_0/A:col10/1733246281322/Put/seqid=0 2024-12-03T17:18:01,985 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68fbb73dc00ff4d04c231a23e30e6f1a#C#compaction#388 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:18:01,985 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/3c92358e2ffb455b99114564e7ae31a1 is 50, key is test_row_0/C:col10/1733246281322/Put/seqid=0 2024-12-03T17:18:02,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742287_1463 (size=12391) 2024-12-03T17:18:02,025 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/03c9ddf2ef0c461288201c51a63206a3 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/03c9ddf2ef0c461288201c51a63206a3 2024-12-03T17:18:02,032 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 68fbb73dc00ff4d04c231a23e30e6f1a/A of 68fbb73dc00ff4d04c231a23e30e6f1a into 03c9ddf2ef0c461288201c51a63206a3(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:18:02,032 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68fbb73dc00ff4d04c231a23e30e6f1a: 2024-12-03T17:18:02,032 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., storeName=68fbb73dc00ff4d04c231a23e30e6f1a/A, priority=13, startTime=1733246281943; duration=0sec 2024-12-03T17:18:02,032 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:18:02,032 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68fbb73dc00ff4d04c231a23e30e6f1a:A 2024-12-03T17:18:02,032 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:18:02,036 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36441 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:18:02,036 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): 68fbb73dc00ff4d04c231a23e30e6f1a/B is initiating minor compaction (all files) 2024-12-03T17:18:02,036 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68fbb73dc00ff4d04c231a23e30e6f1a/B in TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:02,036 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/54ad9b0c5b2c46aab7a67c87671a4b00, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/a069caafc2ea4f6ebfe3ba6711a1d6af, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/e4bdb5853cbb496c9152fda9a0ac0896] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp, totalSize=35.6 K 2024-12-03T17:18:02,038 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 54ad9b0c5b2c46aab7a67c87671a4b00, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733246278686 2024-12-03T17:18:02,043 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting a069caafc2ea4f6ebfe3ba6711a1d6af, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733246279386 2024-12-03T17:18:02,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742288_1464 (size=12391) 2024-12-03T17:18:02,045 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting e4bdb5853cbb496c9152fda9a0ac0896, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733246280611 2024-12-03T17:18:02,064 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/3c92358e2ffb455b99114564e7ae31a1 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/3c92358e2ffb455b99114564e7ae31a1 2024-12-03T17:18:02,071 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68fbb73dc00ff4d04c231a23e30e6f1a#B#compaction#389 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:18:02,071 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/9de49ad238e04e3eaa3b0672225b9a8b is 50, key is test_row_0/B:col10/1733246281322/Put/seqid=0 2024-12-03T17:18:02,092 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 68fbb73dc00ff4d04c231a23e30e6f1a/C of 68fbb73dc00ff4d04c231a23e30e6f1a into 3c92358e2ffb455b99114564e7ae31a1(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:18:02,092 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68fbb73dc00ff4d04c231a23e30e6f1a: 2024-12-03T17:18:02,092 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., storeName=68fbb73dc00ff4d04c231a23e30e6f1a/C, priority=13, startTime=1733246281943; duration=0sec 2024-12-03T17:18:02,092 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:18:02,092 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68fbb73dc00ff4d04c231a23e30e6f1a:C 2024-12-03T17:18:02,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742289_1465 (size=12391) 2024-12-03T17:18:02,120 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/9de49ad238e04e3eaa3b0672225b9a8b as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/9de49ad238e04e3eaa3b0672225b9a8b 2024-12-03T17:18:02,126 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 68fbb73dc00ff4d04c231a23e30e6f1a/B of 68fbb73dc00ff4d04c231a23e30e6f1a into 9de49ad238e04e3eaa3b0672225b9a8b(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:18:02,126 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68fbb73dc00ff4d04c231a23e30e6f1a: 2024-12-03T17:18:02,126 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., storeName=68fbb73dc00ff4d04c231a23e30e6f1a/B, priority=13, startTime=1733246281943; duration=0sec 2024-12-03T17:18:02,127 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:18:02,127 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68fbb73dc00ff4d04c231a23e30e6f1a:B 2024-12-03T17:18:02,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-03T17:18:02,173 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:18:02,174 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T17:18:02,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-12-03T17:18:02,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-03T17:18:02,177 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T17:18:02,178 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T17:18:02,178 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T17:18:02,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 68fbb73dc00ff4d04c231a23e30e6f1a 2024-12-03T17:18:02,179 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 68fbb73dc00ff4d04c231a23e30e6f1a 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-03T17:18:02,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=A 2024-12-03T17:18:02,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:18:02,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=B 2024-12-03T17:18:02,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:18:02,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=C 2024-12-03T17:18:02,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:18:02,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/0522c04bc5fb4b23ac61ef9605e8e7ab is 50, key is test_row_0/A:col10/1733246282177/Put/seqid=0 2024-12-03T17:18:02,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742290_1466 (size=14541) 2024-12-03T17:18:02,212 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/0522c04bc5fb4b23ac61ef9605e8e7ab 2024-12-03T17:18:02,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/e97bfbfb9f7542929a85012dc14740a5 is 50, key is test_row_0/B:col10/1733246282177/Put/seqid=0 2024-12-03T17:18:02,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742291_1467 (size=12151) 2024-12-03T17:18:02,254 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/e97bfbfb9f7542929a85012dc14740a5 2024-12-03T17:18:02,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:02,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246342241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:02,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:02,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:02,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:02,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/aba98252b1884ab8bfe6932d7dd26e8a is 50, key is test_row_0/C:col10/1733246282177/Put/seqid=0 2024-12-03T17:18:02,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:02,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246342253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:02,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:02,269 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:02,269 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:02,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-03T17:18:02,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742292_1468 (size=12151) 2024-12-03T17:18:02,316 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/aba98252b1884ab8bfe6932d7dd26e8a 2024-12-03T17:18:02,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/0522c04bc5fb4b23ac61ef9605e8e7ab as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/0522c04bc5fb4b23ac61ef9605e8e7ab 2024-12-03T17:18:02,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-03T17:18:02,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:02,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. as already flushing 2024-12-03T17:18:02,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:02,334 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:02,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:02,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:02,340 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/0522c04bc5fb4b23ac61ef9605e8e7ab, entries=200, sequenceid=176, filesize=14.2 K 2024-12-03T17:18:02,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/e97bfbfb9f7542929a85012dc14740a5 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/e97bfbfb9f7542929a85012dc14740a5 2024-12-03T17:18:02,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:02,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246342363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:02,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:02,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:02,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:02,367 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/e97bfbfb9f7542929a85012dc14740a5, entries=150, sequenceid=176, filesize=11.9 K 2024-12-03T17:18:02,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/aba98252b1884ab8bfe6932d7dd26e8a as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/aba98252b1884ab8bfe6932d7dd26e8a 2024-12-03T17:18:02,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:02,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246342375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:02,384 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:02,384 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:02,384 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:02,402 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/aba98252b1884ab8bfe6932d7dd26e8a, entries=150, sequenceid=176, filesize=11.9 K 2024-12-03T17:18:02,404 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 68fbb73dc00ff4d04c231a23e30e6f1a in 225ms, sequenceid=176, compaction requested=false 2024-12-03T17:18:02,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 68fbb73dc00ff4d04c231a23e30e6f1a: 2024-12-03T17:18:02,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-03T17:18:02,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-03T17:18:02,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:02,498 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 68fbb73dc00ff4d04c231a23e30e6f1a 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-03T17:18:02,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=A 2024-12-03T17:18:02,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:02,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=B 2024-12-03T17:18:02,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:02,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=C 2024-12-03T17:18:02,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:02,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/1603f971bcf745a59fdb1d5cb915caa4 is 50, key is test_row_0/A:col10/1733246282240/Put/seqid=0 2024-12-03T17:18:02,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742293_1469 (size=12151) 2024-12-03T17:18:02,596 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/1603f971bcf745a59fdb1d5cb915caa4 2024-12-03T17:18:02,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. as already flushing 2024-12-03T17:18:02,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 68fbb73dc00ff4d04c231a23e30e6f1a 2024-12-03T17:18:02,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/a52ac70a46624ab7b6921ba79e784e88 is 50, key is test_row_0/B:col10/1733246282240/Put/seqid=0 2024-12-03T17:18:02,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742294_1470 (size=12151) 2024-12-03T17:18:02,654 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/a52ac70a46624ab7b6921ba79e784e88 2024-12-03T17:18:02,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/2fcb29ebf7ed4bb6866e4c35d0037016 is 50, key is test_row_0/C:col10/1733246282240/Put/seqid=0 2024-12-03T17:18:02,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742295_1471 (size=12151) 2024-12-03T17:18:02,735 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/2fcb29ebf7ed4bb6866e4c35d0037016 2024-12-03T17:18:02,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/1603f971bcf745a59fdb1d5cb915caa4 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/1603f971bcf745a59fdb1d5cb915caa4 2024-12-03T17:18:02,770 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/1603f971bcf745a59fdb1d5cb915caa4, entries=150, sequenceid=187, filesize=11.9 K 2024-12-03T17:18:02,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/a52ac70a46624ab7b6921ba79e784e88 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/a52ac70a46624ab7b6921ba79e784e88 2024-12-03T17:18:02,778 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/a52ac70a46624ab7b6921ba79e784e88, entries=150, sequenceid=187, filesize=11.9 K 2024-12-03T17:18:02,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/2fcb29ebf7ed4bb6866e4c35d0037016 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/2fcb29ebf7ed4bb6866e4c35d0037016 2024-12-03T17:18:02,787 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/2fcb29ebf7ed4bb6866e4c35d0037016, entries=150, sequenceid=187, filesize=11.9 K 2024-12-03T17:18:02,788 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=100.63 KB/103050 for 68fbb73dc00ff4d04c231a23e30e6f1a in 290ms, sequenceid=187, compaction requested=true 2024-12-03T17:18:02,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 68fbb73dc00ff4d04c231a23e30e6f1a: 2024-12-03T17:18:02,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:02,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-12-03T17:18:02,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-12-03T17:18:02,792 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-12-03T17:18:02,792 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 612 msec 2024-12-03T17:18:02,794 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 619 msec 2024-12-03T17:18:02,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 68fbb73dc00ff4d04c231a23e30e6f1a 2024-12-03T17:18:02,800 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 68fbb73dc00ff4d04c231a23e30e6f1a 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-03T17:18:02,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=A 2024-12-03T17:18:02,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:02,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=B 2024-12-03T17:18:02,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:02,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=C 2024-12-03T17:18:02,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:02,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-03T17:18:02,804 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:18:02,805 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T17:18:02,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-12-03T17:18:02,807 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T17:18:02,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-03T17:18:02,807 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T17:18:02,808 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T17:18:02,813 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/daf905ac94ee4333a3dac1e762d41b0e is 50, key is test_row_0/A:col10/1733246282774/Put/seqid=0 2024-12-03T17:18:02,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742296_1472 (size=16931) 2024-12-03T17:18:02,831 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/daf905ac94ee4333a3dac1e762d41b0e 2024-12-03T17:18:02,851 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/cc14cce709a74882ad02255e647e959c is 50, key is test_row_0/B:col10/1733246282774/Put/seqid=0 2024-12-03T17:18:02,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742297_1473 (size=12151) 2024-12-03T17:18:02,859 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/cc14cce709a74882ad02255e647e959c 2024-12-03T17:18:02,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/dce5227af3b24d91a84a99607d5807ed is 50, key is test_row_0/C:col10/1733246282774/Put/seqid=0 2024-12-03T17:18:02,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:02,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246342853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:02,873 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:02,873 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:02,873 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:02,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:02,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246342858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:02,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:02,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:02,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:02,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:02,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246342861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:02,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:02,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:02,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:02,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:02,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246342872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:02,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:02,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:02,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:02,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:02,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48068 deadline: 1733246342873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:02,893 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:02,894 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:02,894 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:02,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742298_1474 (size=12151) 2024-12-03T17:18:02,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-03T17:18:02,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-03T17:18:02,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:02,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. as already flushing 2024-12-03T17:18:02,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:02,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:02,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:02,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:02,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:02,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:02,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246342985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:02,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246342985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:02,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:02,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:02,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:02,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:02,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246342987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:02,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:02,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:02,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246342987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:02,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:02,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:02,999 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:02,999 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:02,999 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:03,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:03,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:03,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:03,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:03,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48068 deadline: 1733246343003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:03,027 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:03,027 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:03,027 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:03,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-03T17:18:03,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-03T17:18:03,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:03,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. as already flushing 2024-12-03T17:18:03,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:03,127 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:03,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:03,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:03,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:03,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246343204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:03,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:03,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246343204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:03,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:03,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:03,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:03,215 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:03,215 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:03,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:03,215 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:03,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246343204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:03,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:03,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:03,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:03,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:03,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246343207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:03,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:03,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:03,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:03,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:03,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48068 deadline: 1733246343235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:03,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:03,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:03,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:03,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-03T17:18:03,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:03,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. as already flushing 2024-12-03T17:18:03,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:03,284 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:03,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:03,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:03,297 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/dce5227af3b24d91a84a99607d5807ed 2024-12-03T17:18:03,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/daf905ac94ee4333a3dac1e762d41b0e as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/daf905ac94ee4333a3dac1e762d41b0e 2024-12-03T17:18:03,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/daf905ac94ee4333a3dac1e762d41b0e, entries=250, sequenceid=207, filesize=16.5 K 2024-12-03T17:18:03,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/cc14cce709a74882ad02255e647e959c as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/cc14cce709a74882ad02255e647e959c 2024-12-03T17:18:03,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/cc14cce709a74882ad02255e647e959c, entries=150, sequenceid=207, filesize=11.9 K 2024-12-03T17:18:03,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/dce5227af3b24d91a84a99607d5807ed as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/dce5227af3b24d91a84a99607d5807ed 2024-12-03T17:18:03,377 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/dce5227af3b24d91a84a99607d5807ed, entries=150, sequenceid=207, filesize=11.9 K 2024-12-03T17:18:03,380 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 68fbb73dc00ff4d04c231a23e30e6f1a in 580ms, sequenceid=207, compaction requested=true 2024-12-03T17:18:03,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 68fbb73dc00ff4d04c231a23e30e6f1a: 2024-12-03T17:18:03,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68fbb73dc00ff4d04c231a23e30e6f1a:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T17:18:03,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:18:03,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68fbb73dc00ff4d04c231a23e30e6f1a:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T17:18:03,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T17:18:03,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68fbb73dc00ff4d04c231a23e30e6f1a:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T17:18:03,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-03T17:18:03,382 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T17:18:03,383 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T17:18:03,384 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 56014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T17:18:03,384 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): 68fbb73dc00ff4d04c231a23e30e6f1a/A is initiating minor compaction (all files) 2024-12-03T17:18:03,384 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68fbb73dc00ff4d04c231a23e30e6f1a/A in TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:03,385 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/03c9ddf2ef0c461288201c51a63206a3, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/0522c04bc5fb4b23ac61ef9605e8e7ab, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/1603f971bcf745a59fdb1d5cb915caa4, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/daf905ac94ee4333a3dac1e762d41b0e] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp, totalSize=54.7 K 2024-12-03T17:18:03,385 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48844 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T17:18:03,385 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): 68fbb73dc00ff4d04c231a23e30e6f1a/C is initiating minor compaction (all files) 2024-12-03T17:18:03,385 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68fbb73dc00ff4d04c231a23e30e6f1a/C in TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:03,385 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/3c92358e2ffb455b99114564e7ae31a1, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/aba98252b1884ab8bfe6932d7dd26e8a, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/2fcb29ebf7ed4bb6866e4c35d0037016, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/dce5227af3b24d91a84a99607d5807ed] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp, totalSize=47.7 K 2024-12-03T17:18:03,385 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03c9ddf2ef0c461288201c51a63206a3, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733246280611 2024-12-03T17:18:03,386 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0522c04bc5fb4b23ac61ef9605e8e7ab, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733246281489 2024-12-03T17:18:03,386 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c92358e2ffb455b99114564e7ae31a1, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733246280611 2024-12-03T17:18:03,386 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1603f971bcf745a59fdb1d5cb915caa4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1733246282221 2024-12-03T17:18:03,387 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting daf905ac94ee4333a3dac1e762d41b0e, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733246282698 2024-12-03T17:18:03,387 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting aba98252b1884ab8bfe6932d7dd26e8a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733246281489 2024-12-03T17:18:03,387 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fcb29ebf7ed4bb6866e4c35d0037016, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1733246282221 2024-12-03T17:18:03,387 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting dce5227af3b24d91a84a99607d5807ed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733246282774 2024-12-03T17:18:03,413 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68fbb73dc00ff4d04c231a23e30e6f1a#C#compaction#399 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:18:03,413 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/c285ac2ead7d43a8a089d3883775c62b is 50, key is test_row_0/C:col10/1733246282774/Put/seqid=0 2024-12-03T17:18:03,414 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68fbb73dc00ff4d04c231a23e30e6f1a#A#compaction#400 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:18:03,416 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/737cd9c702304a3ea414af02c3397960 is 50, key is test_row_0/A:col10/1733246282774/Put/seqid=0 2024-12-03T17:18:03,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-03T17:18:03,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-03T17:18:03,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:03,442 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 68fbb73dc00ff4d04c231a23e30e6f1a 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-03T17:18:03,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=A 2024-12-03T17:18:03,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:03,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=B 2024-12-03T17:18:03,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:03,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=C 2024-12-03T17:18:03,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:03,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742300_1476 (size=12527) 2024-12-03T17:18:03,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742299_1475 (size=12527) 2024-12-03T17:18:03,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/66a2052056f94daaa1582fc84a1750c3 is 50, key is test_row_0/A:col10/1733246282859/Put/seqid=0 2024-12-03T17:18:03,527 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/737cd9c702304a3ea414af02c3397960 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/737cd9c702304a3ea414af02c3397960 2024-12-03T17:18:03,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. as already flushing 2024-12-03T17:18:03,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 68fbb73dc00ff4d04c231a23e30e6f1a 2024-12-03T17:18:03,541 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/c285ac2ead7d43a8a089d3883775c62b as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/c285ac2ead7d43a8a089d3883775c62b 2024-12-03T17:18:03,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742301_1477 (size=12151) 2024-12-03T17:18:03,570 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 68fbb73dc00ff4d04c231a23e30e6f1a/C of 68fbb73dc00ff4d04c231a23e30e6f1a into c285ac2ead7d43a8a089d3883775c62b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:18:03,570 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68fbb73dc00ff4d04c231a23e30e6f1a: 2024-12-03T17:18:03,570 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., storeName=68fbb73dc00ff4d04c231a23e30e6f1a/C, priority=12, startTime=1733246283380; duration=0sec 2024-12-03T17:18:03,570 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:18:03,570 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68fbb73dc00ff4d04c231a23e30e6f1a:C 2024-12-03T17:18:03,570 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T17:18:03,573 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48844 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T17:18:03,573 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): 68fbb73dc00ff4d04c231a23e30e6f1a/B is initiating minor compaction (all files) 2024-12-03T17:18:03,573 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68fbb73dc00ff4d04c231a23e30e6f1a/B in TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:03,573 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/9de49ad238e04e3eaa3b0672225b9a8b, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/e97bfbfb9f7542929a85012dc14740a5, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/a52ac70a46624ab7b6921ba79e784e88, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/cc14cce709a74882ad02255e647e959c] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp, totalSize=47.7 K 2024-12-03T17:18:03,575 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 68fbb73dc00ff4d04c231a23e30e6f1a/A of 68fbb73dc00ff4d04c231a23e30e6f1a into 737cd9c702304a3ea414af02c3397960(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:18:03,575 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68fbb73dc00ff4d04c231a23e30e6f1a: 2024-12-03T17:18:03,576 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., storeName=68fbb73dc00ff4d04c231a23e30e6f1a/A, priority=12, startTime=1733246283380; duration=0sec 2024-12-03T17:18:03,576 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:18:03,576 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68fbb73dc00ff4d04c231a23e30e6f1a:A 2024-12-03T17:18:03,580 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 9de49ad238e04e3eaa3b0672225b9a8b, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733246280611 2024-12-03T17:18:03,583 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting e97bfbfb9f7542929a85012dc14740a5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1733246281489 2024-12-03T17:18:03,586 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting a52ac70a46624ab7b6921ba79e784e88, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1733246282221 2024-12-03T17:18:03,590 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting cc14cce709a74882ad02255e647e959c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733246282774 2024-12-03T17:18:03,649 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68fbb73dc00ff4d04c231a23e30e6f1a#B#compaction#402 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:18:03,649 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/1c993d416a9d45f199b8e05d2b6e03fb is 50, key is test_row_0/B:col10/1733246282774/Put/seqid=0 2024-12-03T17:18:03,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:03,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246343624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:03,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:03,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:03,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:03,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:03,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246343626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:03,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:03,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246343629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:03,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:03,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:03,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:03,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:03,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:03,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:03,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742302_1478 (size=12527) 2024-12-03T17:18:03,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:03,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48068 deadline: 1733246343651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:03,686 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:03,686 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:03,686 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:03,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:03,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246343660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:03,690 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:03,690 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:03,690 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:03,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:03,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246343764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:03,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:03,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:03,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:03,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:03,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246343765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:03,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:03,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:03,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:03,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:03,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246343773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:03,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:03,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:03,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:03,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:03,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48068 deadline: 1733246343794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:03,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:03,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:03,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:03,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:03,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246343796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:03,817 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:03,817 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:03,817 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:03,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-03T17:18:03,947 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/66a2052056f94daaa1582fc84a1750c3 2024-12-03T17:18:03,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/c26bc509add44f7a9e8041aff36ffcc9 is 50, key is test_row_0/B:col10/1733246282859/Put/seqid=0 2024-12-03T17:18:03,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:03,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246343985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:03,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:03,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246343985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:03,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:03,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:03,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:03,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:03,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:03,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:04,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:04,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246344006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:04,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:04,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:04,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:04,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:04,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48068 deadline: 1733246344024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:04,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:04,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246344024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:04,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:04,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:04,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:04,038 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:04,038 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:04,038 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:04,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742303_1479 (size=12151) 2024-12-03T17:18:04,061 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/c26bc509add44f7a9e8041aff36ffcc9 2024-12-03T17:18:04,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/5f0769e4cf50452b806883717e384009 is 50, key is test_row_0/C:col10/1733246282859/Put/seqid=0 2024-12-03T17:18:04,104 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/1c993d416a9d45f199b8e05d2b6e03fb as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/1c993d416a9d45f199b8e05d2b6e03fb 2024-12-03T17:18:04,111 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 68fbb73dc00ff4d04c231a23e30e6f1a/B of 68fbb73dc00ff4d04c231a23e30e6f1a into 1c993d416a9d45f199b8e05d2b6e03fb(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:18:04,111 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68fbb73dc00ff4d04c231a23e30e6f1a: 2024-12-03T17:18:04,111 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., storeName=68fbb73dc00ff4d04c231a23e30e6f1a/B, priority=12, startTime=1733246283380; duration=0sec 2024-12-03T17:18:04,112 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:18:04,112 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68fbb73dc00ff4d04c231a23e30e6f1a:B 2024-12-03T17:18:04,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742304_1480 (size=12151) 2024-12-03T17:18:04,123 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/5f0769e4cf50452b806883717e384009 2024-12-03T17:18:04,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/66a2052056f94daaa1582fc84a1750c3 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/66a2052056f94daaa1582fc84a1750c3 2024-12-03T17:18:04,196 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/66a2052056f94daaa1582fc84a1750c3, entries=150, sequenceid=219, filesize=11.9 K 2024-12-03T17:18:04,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/c26bc509add44f7a9e8041aff36ffcc9 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/c26bc509add44f7a9e8041aff36ffcc9 2024-12-03T17:18:04,226 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/c26bc509add44f7a9e8041aff36ffcc9, entries=150, sequenceid=219, filesize=11.9 K 2024-12-03T17:18:04,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/5f0769e4cf50452b806883717e384009 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/5f0769e4cf50452b806883717e384009 2024-12-03T17:18:04,260 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/5f0769e4cf50452b806883717e384009, entries=150, sequenceid=219, filesize=11.9 K 2024-12-03T17:18:04,262 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 68fbb73dc00ff4d04c231a23e30e6f1a in 820ms, sequenceid=219, compaction requested=false 2024-12-03T17:18:04,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 68fbb73dc00ff4d04c231a23e30e6f1a: 2024-12-03T17:18:04,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:04,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-12-03T17:18:04,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-12-03T17:18:04,294 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-12-03T17:18:04,294 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4740 sec 2024-12-03T17:18:04,297 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.4900 sec 2024-12-03T17:18:04,319 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 68fbb73dc00ff4d04c231a23e30e6f1a 3/3 column families, dataSize=127.47 KB heapSize=379.97 KB 2024-12-03T17:18:04,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=A 2024-12-03T17:18:04,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:18:04,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=B 2024-12-03T17:18:04,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:18:04,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=C 2024-12-03T17:18:04,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:18:04,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 68fbb73dc00ff4d04c231a23e30e6f1a 2024-12-03T17:18:04,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/0531ed71363d4f25a49b8f55c876b9cf is 50, key is test_row_0/A:col10/1733246284313/Put/seqid=0 2024-12-03T17:18:04,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742305_1481 (size=12151) 2024-12-03T17:18:04,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:04,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246344359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:04,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:04,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:04,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:04,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:04,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246344371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:04,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:04,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:04,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:04,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:04,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246344373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:04,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:04,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48068 deadline: 1733246344379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:04,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:04,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246344382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:04,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:04,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:04,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:04,398 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:04,398 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:04,398 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:04,398 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:04,398 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:04,398 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:04,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:04,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246344483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:04,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:04,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:04,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:04,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:04,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246344504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:04,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:04,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246344504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:04,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:04,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:04,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:04,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:04,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246344507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:04,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:04,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:04,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:04,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:04,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48068 deadline: 1733246344507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:04,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:04,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:04,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:04,523 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:04,523 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:04,523 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:04,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:04,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246344707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:04,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:04,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:04,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:04,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:04,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246344724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:04,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:04,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:04,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:04,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:04,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246344734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:04,746 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:04,746 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:04,746 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:04,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:04,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246344734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:04,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:04,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48068 deadline: 1733246344734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:04,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:04,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:04,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:04,748 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:04,748 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:04,748 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:04,782 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/0531ed71363d4f25a49b8f55c876b9cf 2024-12-03T17:18:04,795 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/3018bc5f2f464461a04a8f58e90d37b8 is 50, key is test_row_0/B:col10/1733246284313/Put/seqid=0 2024-12-03T17:18:04,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742306_1482 (size=12151) 2024-12-03T17:18:04,812 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/3018bc5f2f464461a04a8f58e90d37b8 2024-12-03T17:18:04,829 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/a2eb993352eb4ae799d5a11059842bd4 is 50, key is test_row_0/C:col10/1733246284313/Put/seqid=0 2024-12-03T17:18:04,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742307_1483 (size=12151) 2024-12-03T17:18:04,881 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/a2eb993352eb4ae799d5a11059842bd4 2024-12-03T17:18:04,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/0531ed71363d4f25a49b8f55c876b9cf as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/0531ed71363d4f25a49b8f55c876b9cf 2024-12-03T17:18:04,902 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/0531ed71363d4f25a49b8f55c876b9cf, entries=150, sequenceid=244, filesize=11.9 K 2024-12-03T17:18:04,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/3018bc5f2f464461a04a8f58e90d37b8 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/3018bc5f2f464461a04a8f58e90d37b8 2024-12-03T17:18:04,910 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/3018bc5f2f464461a04a8f58e90d37b8, entries=150, sequenceid=244, filesize=11.9 K 2024-12-03T17:18:04,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/a2eb993352eb4ae799d5a11059842bd4 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/a2eb993352eb4ae799d5a11059842bd4 2024-12-03T17:18:04,915 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/a2eb993352eb4ae799d5a11059842bd4, entries=150, sequenceid=244, filesize=11.9 K 2024-12-03T17:18:04,916 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=46.96 KB/48090 for 68fbb73dc00ff4d04c231a23e30e6f1a in 597ms, sequenceid=244, compaction requested=true 2024-12-03T17:18:04,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 68fbb73dc00ff4d04c231a23e30e6f1a: 2024-12-03T17:18:04,916 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:18:04,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68fbb73dc00ff4d04c231a23e30e6f1a:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T17:18:04,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:18:04,917 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:18:04,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68fbb73dc00ff4d04c231a23e30e6f1a:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T17:18:04,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:18:04,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68fbb73dc00ff4d04c231a23e30e6f1a:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T17:18:04,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:18:04,919 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:18:04,920 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): 68fbb73dc00ff4d04c231a23e30e6f1a/A is initiating minor compaction (all files) 2024-12-03T17:18:04,920 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68fbb73dc00ff4d04c231a23e30e6f1a/A in TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:04,920 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/737cd9c702304a3ea414af02c3397960, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/66a2052056f94daaa1582fc84a1750c3, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/0531ed71363d4f25a49b8f55c876b9cf] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp, totalSize=36.0 K 2024-12-03T17:18:04,920 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 737cd9c702304a3ea414af02c3397960, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733246282774 2024-12-03T17:18:04,921 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:18:04,921 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): 68fbb73dc00ff4d04c231a23e30e6f1a/B is initiating minor compaction (all files) 2024-12-03T17:18:04,921 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68fbb73dc00ff4d04c231a23e30e6f1a/B in TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:04,921 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/1c993d416a9d45f199b8e05d2b6e03fb, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/c26bc509add44f7a9e8041aff36ffcc9, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/3018bc5f2f464461a04a8f58e90d37b8] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp, totalSize=36.0 K 2024-12-03T17:18:04,921 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 66a2052056f94daaa1582fc84a1750c3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1733246282857 2024-12-03T17:18:04,922 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c993d416a9d45f199b8e05d2b6e03fb, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733246282774 2024-12-03T17:18:04,922 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 0531ed71363d4f25a49b8f55c876b9cf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733246283603 2024-12-03T17:18:04,922 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting c26bc509add44f7a9e8041aff36ffcc9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1733246282857 2024-12-03T17:18:04,922 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3018bc5f2f464461a04a8f58e90d37b8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733246283603 2024-12-03T17:18:04,930 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68fbb73dc00ff4d04c231a23e30e6f1a#A#compaction#408 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:18:04,930 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/efbc2632ffa94a2ba44d64d0c289e43b is 50, key is test_row_0/A:col10/1733246284313/Put/seqid=0 2024-12-03T17:18:04,939 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68fbb73dc00ff4d04c231a23e30e6f1a#B#compaction#409 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:18:04,940 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/dcf868a27a8d4d978384df27bce8da2a is 50, key is test_row_0/B:col10/1733246284313/Put/seqid=0 2024-12-03T17:18:04,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742308_1484 (size=12629) 2024-12-03T17:18:04,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742309_1485 (size=12629) 2024-12-03T17:18:04,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-03T17:18:04,964 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:18:04,964 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T17:18:04,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-12-03T17:18:04,967 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T17:18:04,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-03T17:18:04,968 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T17:18:04,968 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T17:18:05,030 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 68fbb73dc00ff4d04c231a23e30e6f1a 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-03T17:18:05,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=A 2024-12-03T17:18:05,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:05,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=B 2024-12-03T17:18:05,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:05,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=C 2024-12-03T17:18:05,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:05,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 68fbb73dc00ff4d04c231a23e30e6f1a 2024-12-03T17:18:05,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/cb207c5b3e4944b2b1f83cac126dd576 is 50, key is test_row_0/A:col10/1733246284372/Put/seqid=0 2024-12-03T17:18:05,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742310_1486 (size=14541) 2024-12-03T17:18:05,065 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/cb207c5b3e4944b2b1f83cac126dd576 2024-12-03T17:18:05,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-03T17:18:05,104 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/09f0f96b4e844faf9510dd246d1f0480 is 50, key is test_row_0/B:col10/1733246284372/Put/seqid=0 2024-12-03T17:18:05,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-03T17:18:05,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:05,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. as already flushing 2024-12-03T17:18:05,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:05,126 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:05,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:05,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:05,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742311_1487 (size=12151) 2024-12-03T17:18:05,147 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/09f0f96b4e844faf9510dd246d1f0480 2024-12-03T17:18:05,160 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733246285081, max timestamp=1733246285154Num uniques 859; 2024-12-03T17:18:05,160 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T17:18:05,163 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733246285081, max timestamp=1733246285154Num uniques 859; 2024-12-03T17:18:05,163 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T17:18:05,180 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/c3da1a61f742433b982e263845249ba4 is 50, key is test_row_0/C:col10/1733246284372/Put/seqid=0 2024-12-03T17:18:05,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742312_1488 (size=12151) 2024-12-03T17:18:05,216 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/c3da1a61f742433b982e263845249ba4 2024-12-03T17:18:05,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:05,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246345189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:05,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:05,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:05,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:05,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:05,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48068 deadline: 1733246345195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:05,223 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:05,223 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:05,223 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:05,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/cb207c5b3e4944b2b1f83cac126dd576 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/cb207c5b3e4944b2b1f83cac126dd576 2024-12-03T17:18:05,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:05,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246345212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:05,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:05,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246345215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:05,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:05,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246345217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:05,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:05,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:05,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:05,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:05,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:05,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:05,258 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:05,259 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:05,259 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:05,268 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/cb207c5b3e4944b2b1f83cac126dd576, entries=200, sequenceid=255, filesize=14.2 K 2024-12-03T17:18:05,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/09f0f96b4e844faf9510dd246d1f0480 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/09f0f96b4e844faf9510dd246d1f0480 2024-12-03T17:18:05,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/09f0f96b4e844faf9510dd246d1f0480, entries=150, sequenceid=255, filesize=11.9 K 2024-12-03T17:18:05,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/c3da1a61f742433b982e263845249ba4 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/c3da1a61f742433b982e263845249ba4 2024-12-03T17:18:05,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-03T17:18:05,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:05,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. as already flushing 2024-12-03T17:18:05,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:05,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:05,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:05,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:05,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-03T17:18:05,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/c3da1a61f742433b982e263845249ba4, entries=150, sequenceid=255, filesize=11.9 K 2024-12-03T17:18:05,301 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 68fbb73dc00ff4d04c231a23e30e6f1a in 270ms, sequenceid=255, compaction requested=true 2024-12-03T17:18:05,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 68fbb73dc00ff4d04c231a23e30e6f1a: 2024-12-03T17:18:05,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68fbb73dc00ff4d04c231a23e30e6f1a:A, priority=-2147483648, current under compaction store size is 3 2024-12-03T17:18:05,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T17:18:05,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68fbb73dc00ff4d04c231a23e30e6f1a:B, priority=-2147483648, current under compaction store size is 3 2024-12-03T17:18:05,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-03T17:18:05,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68fbb73dc00ff4d04c231a23e30e6f1a:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T17:18:05,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-12-03T17:18:05,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 68fbb73dc00ff4d04c231a23e30e6f1a 2024-12-03T17:18:05,336 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 68fbb73dc00ff4d04c231a23e30e6f1a 3/3 column families, dataSize=134.18 KB heapSize=383.70 KB 2024-12-03T17:18:05,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=A 2024-12-03T17:18:05,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:18:05,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=B 2024-12-03T17:18:05,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:18:05,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=C 2024-12-03T17:18:05,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:18:05,345 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/dd4169ab47c74a1d9064f18d1f81f89e is 50, key is test_row_0/A:col10/1733246285181/Put/seqid=0 2024-12-03T17:18:05,360 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/efbc2632ffa94a2ba44d64d0c289e43b as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/efbc2632ffa94a2ba44d64d0c289e43b 2024-12-03T17:18:05,362 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/dcf868a27a8d4d978384df27bce8da2a as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/dcf868a27a8d4d978384df27bce8da2a 2024-12-03T17:18:05,365 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 68fbb73dc00ff4d04c231a23e30e6f1a/A of 68fbb73dc00ff4d04c231a23e30e6f1a into efbc2632ffa94a2ba44d64d0c289e43b(size=12.3 K), total size for store is 26.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:18:05,365 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68fbb73dc00ff4d04c231a23e30e6f1a: 2024-12-03T17:18:05,365 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., storeName=68fbb73dc00ff4d04c231a23e30e6f1a/A, priority=13, startTime=1733246284916; duration=0sec 2024-12-03T17:18:05,365 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-12-03T17:18:05,365 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68fbb73dc00ff4d04c231a23e30e6f1a:A 2024-12-03T17:18:05,366 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68fbb73dc00ff4d04c231a23e30e6f1a:A 2024-12-03T17:18:05,366 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T17:18:05,368 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 68fbb73dc00ff4d04c231a23e30e6f1a/B of 68fbb73dc00ff4d04c231a23e30e6f1a into dcf868a27a8d4d978384df27bce8da2a(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:18:05,368 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T17:18:05,368 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68fbb73dc00ff4d04c231a23e30e6f1a: 2024-12-03T17:18:05,368 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): 68fbb73dc00ff4d04c231a23e30e6f1a/C is initiating minor compaction (all files) 2024-12-03T17:18:05,368 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., storeName=68fbb73dc00ff4d04c231a23e30e6f1a/B, priority=13, startTime=1733246284917; duration=0sec 2024-12-03T17:18:05,368 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68fbb73dc00ff4d04c231a23e30e6f1a/C in TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:05,368 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-03T17:18:05,368 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68fbb73dc00ff4d04c231a23e30e6f1a:B 2024-12-03T17:18:05,368 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68fbb73dc00ff4d04c231a23e30e6f1a:B 2024-12-03T17:18:05,368 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/c285ac2ead7d43a8a089d3883775c62b, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/5f0769e4cf50452b806883717e384009, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/a2eb993352eb4ae799d5a11059842bd4, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/c3da1a61f742433b982e263845249ba4] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp, totalSize=47.8 K 2024-12-03T17:18:05,368 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 4 compacting, 0 eligible, 16 blocking 2024-12-03T17:18:05,368 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-03T17:18:05,368 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting c285ac2ead7d43a8a089d3883775c62b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733246282774 2024-12-03T17:18:05,368 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f0769e4cf50452b806883717e384009, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1733246282857 2024-12-03T17:18:05,369 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting a2eb993352eb4ae799d5a11059842bd4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733246283603 2024-12-03T17:18:05,369 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-03T17:18:05,369 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. because compaction request was cancelled 2024-12-03T17:18:05,369 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68fbb73dc00ff4d04c231a23e30e6f1a:C 2024-12-03T17:18:05,369 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-03T17:18:05,370 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting c3da1a61f742433b982e263845249ba4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733246284356 2024-12-03T17:18:05,370 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-03T17:18:05,370 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-03T17:18:05,370 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. because compaction request was cancelled 2024-12-03T17:18:05,370 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68fbb73dc00ff4d04c231a23e30e6f1a:B 2024-12-03T17:18:05,370 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-03T17:18:05,371 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-03T17:18:05,371 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-03T17:18:05,371 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. because compaction request was cancelled 2024-12-03T17:18:05,371 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68fbb73dc00ff4d04c231a23e30e6f1a:A 2024-12-03T17:18:05,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742313_1489 (size=14741) 2024-12-03T17:18:05,388 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68fbb73dc00ff4d04c231a23e30e6f1a#C#compaction#414 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:18:05,388 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/b27940cb751843248d0d82d5f6f54df0 is 50, key is test_row_0/C:col10/1733246284372/Put/seqid=0 2024-12-03T17:18:05,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:05,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246345372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:05,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:05,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:05,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246345373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:05,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:05,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:05,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:05,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48068 deadline: 1733246345374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:05,398 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:05,398 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:05,398 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:05,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:05,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246345378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:05,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:05,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:05,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:05,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:05,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:05,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:05,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:05,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246345382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:05,403 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:05,403 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:05,403 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:05,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742314_1490 (size=12663) 2024-12-03T17:18:05,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-03T17:18:05,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:05,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. as already flushing 2024-12-03T17:18:05,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:05,447 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:05,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:05,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:05,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:05,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246345504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:05,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:05,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246345504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:05,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:05,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:05,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48068 deadline: 1733246345504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:05,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:05,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:05,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:05,522 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:05,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:05,522 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:05,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:05,522 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:05,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:05,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246345504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:05,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:05,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:05,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:05,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:05,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246345514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:05,524 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:05,524 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:05,524 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:05,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-03T17:18:05,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-03T17:18:05,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:05,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. as already flushing 2024-12-03T17:18:05,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:05,607 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:05,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:05,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:05,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:05,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48068 deadline: 1733246345736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:05,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:05,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:05,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:05,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:05,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246345736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:05,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:05,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246345736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:05,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:05,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246345736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:05,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:05,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:05,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:05,747 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:05,747 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:05,747 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:05,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:05,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:05,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:05,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:05,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246345737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:05,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:05,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:05,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:05,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-03T17:18:05,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:05,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. as already flushing 2024-12-03T17:18:05,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:05,762 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:05,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:05,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:05,783 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/dd4169ab47c74a1d9064f18d1f81f89e 2024-12-03T17:18:05,816 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/bfffba2e0e0a4b85aad65f7891a8300c is 50, key is test_row_0/B:col10/1733246285181/Put/seqid=0 2024-12-03T17:18:05,825 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/b27940cb751843248d0d82d5f6f54df0 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/b27940cb751843248d0d82d5f6f54df0 2024-12-03T17:18:05,842 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 68fbb73dc00ff4d04c231a23e30e6f1a/C of 68fbb73dc00ff4d04c231a23e30e6f1a into b27940cb751843248d0d82d5f6f54df0(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:18:05,842 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68fbb73dc00ff4d04c231a23e30e6f1a: 2024-12-03T17:18:05,842 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., storeName=68fbb73dc00ff4d04c231a23e30e6f1a/C, priority=12, startTime=1733246285301; duration=0sec 2024-12-03T17:18:05,842 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:18:05,843 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68fbb73dc00ff4d04c231a23e30e6f1a:C 2024-12-03T17:18:05,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742315_1491 (size=12301) 2024-12-03T17:18:05,859 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/bfffba2e0e0a4b85aad65f7891a8300c 2024-12-03T17:18:05,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/ebc4f629ea6245a78ed2310b2c498a83 is 50, key is test_row_0/C:col10/1733246285181/Put/seqid=0 2024-12-03T17:18:05,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-03T17:18:05,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:05,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. as already flushing 2024-12-03T17:18:05,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:05,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:05,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:05,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:05,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742316_1492 (size=12301) 2024-12-03T17:18:06,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:06,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48068 deadline: 1733246346055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:06,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:06,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246346055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:06,071 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:06,071 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:06,071 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:06,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:06,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246346055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:06,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:06,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:06,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246346056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:06,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:06,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:06,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:06,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:06,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:06,074 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:06,074 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:06,074 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:06,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:06,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246346066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:06,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:06,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:06,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:06,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-03T17:18:06,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:06,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. as already flushing 2024-12-03T17:18:06,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:06,087 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:06,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:06,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:06,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-03T17:18:06,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-03T17:18:06,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:06,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. as already flushing 2024-12-03T17:18:06,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:06,251 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:06,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:06,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:06,338 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/ebc4f629ea6245a78ed2310b2c498a83 2024-12-03T17:18:06,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/dd4169ab47c74a1d9064f18d1f81f89e as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/dd4169ab47c74a1d9064f18d1f81f89e 2024-12-03T17:18:06,354 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/dd4169ab47c74a1d9064f18d1f81f89e, entries=200, sequenceid=286, filesize=14.4 K 2024-12-03T17:18:06,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/bfffba2e0e0a4b85aad65f7891a8300c as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/bfffba2e0e0a4b85aad65f7891a8300c 2024-12-03T17:18:06,369 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/bfffba2e0e0a4b85aad65f7891a8300c, entries=150, sequenceid=286, filesize=12.0 K 2024-12-03T17:18:06,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/ebc4f629ea6245a78ed2310b2c498a83 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/ebc4f629ea6245a78ed2310b2c498a83 2024-12-03T17:18:06,377 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/ebc4f629ea6245a78ed2310b2c498a83, entries=150, sequenceid=286, filesize=12.0 K 2024-12-03T17:18:06,380 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~383.66 KB/392864, currentSize=46.96 KB/48090 for 68fbb73dc00ff4d04c231a23e30e6f1a in 1044ms, sequenceid=286, compaction requested=true 2024-12-03T17:18:06,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 68fbb73dc00ff4d04c231a23e30e6f1a: 2024-12-03T17:18:06,380 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:18:06,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68fbb73dc00ff4d04c231a23e30e6f1a:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T17:18:06,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:18:06,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68fbb73dc00ff4d04c231a23e30e6f1a:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T17:18:06,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:18:06,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68fbb73dc00ff4d04c231a23e30e6f1a:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T17:18:06,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:18:06,381 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:18:06,382 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41911 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:18:06,382 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): 68fbb73dc00ff4d04c231a23e30e6f1a/A is initiating minor compaction (all files) 2024-12-03T17:18:06,382 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68fbb73dc00ff4d04c231a23e30e6f1a/A in TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:06,382 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/efbc2632ffa94a2ba44d64d0c289e43b, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/cb207c5b3e4944b2b1f83cac126dd576, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/dd4169ab47c74a1d9064f18d1f81f89e] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp, totalSize=40.9 K 2024-12-03T17:18:06,384 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37081 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:18:06,384 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): 68fbb73dc00ff4d04c231a23e30e6f1a/B is initiating minor compaction (all files) 2024-12-03T17:18:06,384 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68fbb73dc00ff4d04c231a23e30e6f1a/B in TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:06,384 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/dcf868a27a8d4d978384df27bce8da2a, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/09f0f96b4e844faf9510dd246d1f0480, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/bfffba2e0e0a4b85aad65f7891a8300c] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp, totalSize=36.2 K 2024-12-03T17:18:06,384 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting efbc2632ffa94a2ba44d64d0c289e43b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733246283603 2024-12-03T17:18:06,385 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting dcf868a27a8d4d978384df27bce8da2a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733246283603 2024-12-03T17:18:06,385 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb207c5b3e4944b2b1f83cac126dd576, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733246284356 2024-12-03T17:18:06,389 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 09f0f96b4e844faf9510dd246d1f0480, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733246284356 2024-12-03T17:18:06,389 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd4169ab47c74a1d9064f18d1f81f89e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733246285181 2024-12-03T17:18:06,392 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting bfffba2e0e0a4b85aad65f7891a8300c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733246285181 2024-12-03T17:18:06,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-03T17:18:06,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:06,407 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 68fbb73dc00ff4d04c231a23e30e6f1a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-03T17:18:06,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=A 2024-12-03T17:18:06,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:06,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=B 2024-12-03T17:18:06,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:06,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=C 2024-12-03T17:18:06,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:06,412 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68fbb73dc00ff4d04c231a23e30e6f1a#B#compaction#417 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:18:06,412 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/c33f9d22233941c1aa17132a89624ba6 is 50, key is test_row_0/B:col10/1733246285181/Put/seqid=0 2024-12-03T17:18:06,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/e9a277d43db14f5ca3759debbd83438e is 50, key is test_row_0/A:col10/1733246285371/Put/seqid=0 2024-12-03T17:18:06,422 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68fbb73dc00ff4d04c231a23e30e6f1a#A#compaction#418 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:18:06,423 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/0be604b2c044496b9c0f9b6c2bce3ae5 is 50, key is test_row_0/A:col10/1733246285181/Put/seqid=0 2024-12-03T17:18:06,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742318_1494 (size=12301) 2024-12-03T17:18:06,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742317_1493 (size=12881) 2024-12-03T17:18:06,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742319_1495 (size=12881) 2024-12-03T17:18:06,474 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/0be604b2c044496b9c0f9b6c2bce3ae5 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/0be604b2c044496b9c0f9b6c2bce3ae5 2024-12-03T17:18:06,498 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 68fbb73dc00ff4d04c231a23e30e6f1a/A of 68fbb73dc00ff4d04c231a23e30e6f1a into 0be604b2c044496b9c0f9b6c2bce3ae5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:18:06,498 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68fbb73dc00ff4d04c231a23e30e6f1a: 2024-12-03T17:18:06,498 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., storeName=68fbb73dc00ff4d04c231a23e30e6f1a/A, priority=13, startTime=1733246286380; duration=0sec 2024-12-03T17:18:06,498 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:18:06,498 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68fbb73dc00ff4d04c231a23e30e6f1a:A 2024-12-03T17:18:06,498 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-03T17:18:06,504 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-03T17:18:06,504 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-03T17:18:06,504 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. because compaction request was cancelled 2024-12-03T17:18:06,504 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68fbb73dc00ff4d04c231a23e30e6f1a:C 2024-12-03T17:18:06,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 68fbb73dc00ff4d04c231a23e30e6f1a 2024-12-03T17:18:06,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. as already flushing 2024-12-03T17:18:06,631 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733246286606, max timestamp=1733246286628Num uniques 859; 2024-12-03T17:18:06,631 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T17:18:06,632 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733246286606, max timestamp=1733246286628Num uniques 859; 2024-12-03T17:18:06,632 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733246286606, max timestamp=1733246286628Num uniques 859; 2024-12-03T17:18:06,632 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T17:18:06,632 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T17:18:06,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:06,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246346669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:06,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:06,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:06,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246346683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:06,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:06,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:06,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:06,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:06,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:06,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:06,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246346691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:06,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:06,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:06,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:06,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:06,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246346695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:06,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:06,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:06,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:06,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:06,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48068 deadline: 1733246346703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:06,720 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:06,720 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:06,720 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:06,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:06,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246346805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:06,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:06,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:06,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:06,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:06,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246346810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:06,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:06,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246346814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:06,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:06,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:06,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:06,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:06,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:06,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:06,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:06,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246346824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:06,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:06,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48068 deadline: 1733246346824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:06,839 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:06,839 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:06,839 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:06,839 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:06,839 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:06,839 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:06,866 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/e9a277d43db14f5ca3759debbd83438e 2024-12-03T17:18:06,895 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/c33f9d22233941c1aa17132a89624ba6 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/c33f9d22233941c1aa17132a89624ba6 2024-12-03T17:18:06,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/6a982eda020a4844bc26540a674cfd20 is 50, key is test_row_0/B:col10/1733246285371/Put/seqid=0 2024-12-03T17:18:06,902 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 68fbb73dc00ff4d04c231a23e30e6f1a/B of 68fbb73dc00ff4d04c231a23e30e6f1a into c33f9d22233941c1aa17132a89624ba6(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:18:06,902 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68fbb73dc00ff4d04c231a23e30e6f1a: 2024-12-03T17:18:06,902 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., storeName=68fbb73dc00ff4d04c231a23e30e6f1a/B, priority=13, startTime=1733246286381; duration=0sec 2024-12-03T17:18:06,902 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:18:06,902 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68fbb73dc00ff4d04c231a23e30e6f1a:B 2024-12-03T17:18:06,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742320_1496 (size=12301) 2024-12-03T17:18:07,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:07,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246347034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:07,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:07,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:07,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246347034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:07,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:07,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:07,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:07,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246347035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:07,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:07,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:07,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:07,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:07,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48068 deadline: 1733246347043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:07,047 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:07,047 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:07,047 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:07,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:07,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246347043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:07,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:07,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:07,047 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:07,047 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:07,047 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:07,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:07,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-03T17:18:07,348 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/6a982eda020a4844bc26540a674cfd20 2024-12-03T17:18:07,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/1810a539ba6842b094258a273a218e43 is 50, key is test_row_0/C:col10/1733246285371/Put/seqid=0 2024-12-03T17:18:07,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:07,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246347355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:07,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:07,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48068 deadline: 1733246347355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:07,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:07,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:07,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:07,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:07,372 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:07,372 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:07,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246347356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:07,372 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:07,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:07,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246347356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:07,373 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:07,373 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:07,373 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:07,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:07,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246347356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:07,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:07,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:07,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:07,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:07,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:07,373 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:07,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742321_1497 (size=12301) 2024-12-03T17:18:07,396 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/1810a539ba6842b094258a273a218e43 2024-12-03T17:18:07,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/e9a277d43db14f5ca3759debbd83438e as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/e9a277d43db14f5ca3759debbd83438e 2024-12-03T17:18:07,406 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/e9a277d43db14f5ca3759debbd83438e, entries=150, sequenceid=299, filesize=12.0 K 2024-12-03T17:18:07,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/6a982eda020a4844bc26540a674cfd20 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/6a982eda020a4844bc26540a674cfd20 2024-12-03T17:18:07,411 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/6a982eda020a4844bc26540a674cfd20, entries=150, sequenceid=299, filesize=12.0 K 2024-12-03T17:18:07,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/1810a539ba6842b094258a273a218e43 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/1810a539ba6842b094258a273a218e43 2024-12-03T17:18:07,415 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/1810a539ba6842b094258a273a218e43, entries=150, sequenceid=299, filesize=12.0 K 2024-12-03T17:18:07,416 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 68fbb73dc00ff4d04c231a23e30e6f1a in 1009ms, sequenceid=299, compaction requested=true 2024-12-03T17:18:07,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 68fbb73dc00ff4d04c231a23e30e6f1a: 2024-12-03T17:18:07,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:07,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-12-03T17:18:07,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-12-03T17:18:07,422 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-12-03T17:18:07,424 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 2.4590 sec 2024-12-03T17:18:07,424 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4520 sec 2024-12-03T17:18:07,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733246286661, max timestamp=1733246287897Num uniques 1068; 2024-12-03T17:18:07,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-03T17:18:07,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733246286661, max timestamp=1733246287897Num uniques 1068; 2024-12-03T17:18:07,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-03T17:18:07,904 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733246286661, max timestamp=1733246287897Num uniques 1068; 2024-12-03T17:18:07,904 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-03T17:18:07,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 68fbb73dc00ff4d04c231a23e30e6f1a 2024-12-03T17:18:07,911 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 68fbb73dc00ff4d04c231a23e30e6f1a 3/3 column families, dataSize=53.67 KB heapSize=140.39 KB 2024-12-03T17:18:07,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=A 2024-12-03T17:18:07,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:18:07,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=B 2024-12-03T17:18:07,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:18:07,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=C 2024-12-03T17:18:07,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:18:07,925 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/99a8270564084775be049cd81f479855 is 50, key is test_row_0/A:col10/1733246287897/Put/seqid=0 2024-12-03T17:18:07,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742322_1498 (size=17181) 2024-12-03T17:18:08,023 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733246288001, max timestamp=1733246288019Num uniques 859; 2024-12-03T17:18:08,023 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T17:18:08,023 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733246288001, max timestamp=1733246288019Num uniques 859; 2024-12-03T17:18:08,023 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T17:18:08,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733246288001, max timestamp=1733246288019Num uniques 859; 2024-12-03T17:18:08,024 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T17:18:08,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:08,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246348100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:08,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:08,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:08,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:08,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:08,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246348100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:08,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:08,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:08,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:08,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:08,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246348101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:08,131 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:08,131 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:08,131 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:08,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:08,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246348122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:08,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:08,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:08,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:08,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48068 deadline: 1733246348124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:08,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:08,148 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:08,148 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:08,148 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:08,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:08,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246348234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:08,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:08,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246348234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:08,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:08,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246348234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:08,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:08,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:08,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:08,244 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:08,244 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:08,244 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:08,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:08,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:08,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:08,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:08,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246348254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:08,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:08,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:08,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:08,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:08,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48068 deadline: 1733246348254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:08,271 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:08,271 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:08,271 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:08,365 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/99a8270564084775be049cd81f479855 2024-12-03T17:18:08,385 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/ec5869825b02403caf4acafdf9cd6f07 is 50, key is test_row_0/B:col10/1733246287897/Put/seqid=0 2024-12-03T17:18:08,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742323_1499 (size=12301) 2024-12-03T17:18:08,450 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/ec5869825b02403caf4acafdf9cd6f07 2024-12-03T17:18:08,468 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/b2c89ab0fe2744f08e88a5f13cdca7e8 is 50, key is test_row_0/C:col10/1733246287897/Put/seqid=0 2024-12-03T17:18:08,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:08,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246348454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:08,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:08,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:08,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:08,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:08,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246348454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:08,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:08,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:08,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246348454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:08,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:08,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:08,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:08,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:08,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:08,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:08,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246348473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:08,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:08,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48068 deadline: 1733246348473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:08,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:08,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:08,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:08,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:08,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:08,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:08,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742324_1500 (size=12301) 2024-12-03T17:18:08,514 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/b2c89ab0fe2744f08e88a5f13cdca7e8 2024-12-03T17:18:08,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/99a8270564084775be049cd81f479855 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/99a8270564084775be049cd81f479855 2024-12-03T17:18:08,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/99a8270564084775be049cd81f479855, entries=250, sequenceid=342, filesize=16.8 K 2024-12-03T17:18:08,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/ec5869825b02403caf4acafdf9cd6f07 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/ec5869825b02403caf4acafdf9cd6f07 2024-12-03T17:18:08,534 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/ec5869825b02403caf4acafdf9cd6f07, entries=150, sequenceid=342, filesize=12.0 K 2024-12-03T17:18:08,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/b2c89ab0fe2744f08e88a5f13cdca7e8 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/b2c89ab0fe2744f08e88a5f13cdca7e8 2024-12-03T17:18:08,549 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/b2c89ab0fe2744f08e88a5f13cdca7e8, entries=150, sequenceid=342, filesize=12.0 K 2024-12-03T17:18:08,550 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~140.34 KB/143712, currentSize=134.18 KB/137400 for 68fbb73dc00ff4d04c231a23e30e6f1a in 639ms, sequenceid=342, compaction requested=true 2024-12-03T17:18:08,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 68fbb73dc00ff4d04c231a23e30e6f1a: 2024-12-03T17:18:08,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68fbb73dc00ff4d04c231a23e30e6f1a:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T17:18:08,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:18:08,550 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:18:08,550 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:18:08,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68fbb73dc00ff4d04c231a23e30e6f1a:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T17:18:08,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:18:08,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68fbb73dc00ff4d04c231a23e30e6f1a:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T17:18:08,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:18:08,555 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:18:08,555 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): 68fbb73dc00ff4d04c231a23e30e6f1a/B is initiating minor compaction (all files) 2024-12-03T17:18:08,555 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68fbb73dc00ff4d04c231a23e30e6f1a/B in TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:08,555 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/c33f9d22233941c1aa17132a89624ba6, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/6a982eda020a4844bc26540a674cfd20, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/ec5869825b02403caf4acafdf9cd6f07] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp, totalSize=36.6 K 2024-12-03T17:18:08,556 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42363 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:18:08,556 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): 68fbb73dc00ff4d04c231a23e30e6f1a/A is initiating minor compaction (all files) 2024-12-03T17:18:08,556 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68fbb73dc00ff4d04c231a23e30e6f1a/A in TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:08,556 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/0be604b2c044496b9c0f9b6c2bce3ae5, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/e9a277d43db14f5ca3759debbd83438e, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/99a8270564084775be049cd81f479855] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp, totalSize=41.4 K 2024-12-03T17:18:08,556 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting c33f9d22233941c1aa17132a89624ba6, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733246285181 2024-12-03T17:18:08,557 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a982eda020a4844bc26540a674cfd20, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1733246285371 2024-12-03T17:18:08,557 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0be604b2c044496b9c0f9b6c2bce3ae5, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733246285181 2024-12-03T17:18:08,557 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting ec5869825b02403caf4acafdf9cd6f07, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1733246287897 2024-12-03T17:18:08,557 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9a277d43db14f5ca3759debbd83438e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1733246285371 2024-12-03T17:18:08,558 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99a8270564084775be049cd81f479855, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1733246287897 2024-12-03T17:18:08,579 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68fbb73dc00ff4d04c231a23e30e6f1a#A#compaction#425 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:18:08,580 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/f1a4a6efba204836ac341121e0dd5a27 is 50, key is test_row_0/A:col10/1733246287897/Put/seqid=0 2024-12-03T17:18:08,591 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68fbb73dc00ff4d04c231a23e30e6f1a#B#compaction#426 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:18:08,592 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/a2f3ced2d96f4f5c8e036f6daa544495 is 50, key is test_row_0/B:col10/1733246287897/Put/seqid=0 2024-12-03T17:18:08,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742325_1501 (size=12983) 2024-12-03T17:18:08,641 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/f1a4a6efba204836ac341121e0dd5a27 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/f1a4a6efba204836ac341121e0dd5a27 2024-12-03T17:18:08,646 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 68fbb73dc00ff4d04c231a23e30e6f1a/A of 68fbb73dc00ff4d04c231a23e30e6f1a into f1a4a6efba204836ac341121e0dd5a27(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:18:08,647 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68fbb73dc00ff4d04c231a23e30e6f1a: 2024-12-03T17:18:08,647 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., storeName=68fbb73dc00ff4d04c231a23e30e6f1a/A, priority=13, startTime=1733246288550; duration=0sec 2024-12-03T17:18:08,647 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:18:08,647 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68fbb73dc00ff4d04c231a23e30e6f1a:A 2024-12-03T17:18:08,647 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T17:18:08,649 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49566 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T17:18:08,650 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): 68fbb73dc00ff4d04c231a23e30e6f1a/C is initiating minor compaction (all files) 2024-12-03T17:18:08,650 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68fbb73dc00ff4d04c231a23e30e6f1a/C in TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:08,650 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/b27940cb751843248d0d82d5f6f54df0, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/ebc4f629ea6245a78ed2310b2c498a83, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/1810a539ba6842b094258a273a218e43, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/b2c89ab0fe2744f08e88a5f13cdca7e8] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp, totalSize=48.4 K 2024-12-03T17:18:08,652 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting b27940cb751843248d0d82d5f6f54df0, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1733246284356 2024-12-03T17:18:08,652 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebc4f629ea6245a78ed2310b2c498a83, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733246285181 2024-12-03T17:18:08,653 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1810a539ba6842b094258a273a218e43, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1733246285371 2024-12-03T17:18:08,653 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2c89ab0fe2744f08e88a5f13cdca7e8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1733246287897 2024-12-03T17:18:08,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742326_1502 (size=12983) 2024-12-03T17:18:08,675 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68fbb73dc00ff4d04c231a23e30e6f1a#C#compaction#427 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:18:08,676 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/9f7113faa3b74899b5027b0e70049639 is 50, key is test_row_0/C:col10/1733246287897/Put/seqid=0 2024-12-03T17:18:08,678 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/a2f3ced2d96f4f5c8e036f6daa544495 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/a2f3ced2d96f4f5c8e036f6daa544495 2024-12-03T17:18:08,683 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 68fbb73dc00ff4d04c231a23e30e6f1a/B of 68fbb73dc00ff4d04c231a23e30e6f1a into a2f3ced2d96f4f5c8e036f6daa544495(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:18:08,683 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68fbb73dc00ff4d04c231a23e30e6f1a: 2024-12-03T17:18:08,683 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., storeName=68fbb73dc00ff4d04c231a23e30e6f1a/B, priority=13, startTime=1733246288550; duration=0sec 2024-12-03T17:18:08,683 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:18:08,684 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68fbb73dc00ff4d04c231a23e30e6f1a:B 2024-12-03T17:18:08,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742327_1503 (size=12949) 2024-12-03T17:18:08,734 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/9f7113faa3b74899b5027b0e70049639 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/9f7113faa3b74899b5027b0e70049639 2024-12-03T17:18:08,739 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 68fbb73dc00ff4d04c231a23e30e6f1a/C of 68fbb73dc00ff4d04c231a23e30e6f1a into 9f7113faa3b74899b5027b0e70049639(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:18:08,739 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68fbb73dc00ff4d04c231a23e30e6f1a: 2024-12-03T17:18:08,740 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., storeName=68fbb73dc00ff4d04c231a23e30e6f1a/C, priority=12, startTime=1733246288550; duration=0sec 2024-12-03T17:18:08,740 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:18:08,740 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68fbb73dc00ff4d04c231a23e30e6f1a:C 2024-12-03T17:18:08,801 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733246288092, max timestamp=1733246288795Num uniques 1018; 2024-12-03T17:18:08,801 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733246288092, max timestamp=1733246288795Num uniques 1018; 2024-12-03T17:18:08,801 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-03T17:18:08,801 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-03T17:18:08,802 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 68fbb73dc00ff4d04c231a23e30e6f1a 3/3 column families, dataSize=64.85 KB heapSize=178.80 KB 2024-12-03T17:18:08,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=A 2024-12-03T17:18:08,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:18:08,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=B 2024-12-03T17:18:08,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:18:08,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=C 2024-12-03T17:18:08,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-03T17:18:08,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 68fbb73dc00ff4d04c231a23e30e6f1a 2024-12-03T17:18:08,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/a1ccb970015941188cd39a7d905fabf9 is 50, key is test_row_0/A:col10/1733246288099/Put/seqid=0 2024-12-03T17:18:08,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742328_1504 (size=14741) 2024-12-03T17:18:08,826 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/a1ccb970015941188cd39a7d905fabf9 2024-12-03T17:18:08,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/f7654c94e6544e30b5652f03efd02ef8 is 50, key is test_row_0/B:col10/1733246288099/Put/seqid=0 2024-12-03T17:18:08,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742329_1505 (size=12301) 2024-12-03T17:18:08,841 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/f7654c94e6544e30b5652f03efd02ef8 2024-12-03T17:18:08,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/ca629c3ec3a14522862ea2f5a863d087 is 50, key is test_row_0/C:col10/1733246288099/Put/seqid=0 2024-12-03T17:18:08,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742330_1506 (size=12301) 2024-12-03T17:18:08,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:08,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48068 deadline: 1733246348885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:08,901 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:08,901 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:08,901 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:08,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733246288819, max timestamp=1733246288900Num uniques 859; 2024-12-03T17:18:08,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T17:18:08,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733246288819, max timestamp=1733246288900Num uniques 859; 2024-12-03T17:18:08,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733246288819, max timestamp=1733246288900Num uniques 859; 2024-12-03T17:18:08,904 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T17:18:08,904 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-03T17:18:08,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:08,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246348946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:08,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:08,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246348947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:08,965 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:08,965 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:08,965 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:08,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:08,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:08,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:08,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:08,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246348951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:08,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:08,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:08,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246348953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:08,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:08,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:08,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:08,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:08,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:09,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:09,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48068 deadline: 1733246349004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:09,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:09,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:09,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:09,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:09,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246349073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:09,086 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:09,086 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:09,086 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:09,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:09,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246349074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:09,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:09,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246349074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:09,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:09,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:09,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246349074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:09,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:09,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:09,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:09,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:09,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:09,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:09,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:09,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:09,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-03T17:18:09,164 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:18:09,164 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T17:18:09,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-12-03T17:18:09,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-03T17:18:09,167 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T17:18:09,169 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T17:18:09,169 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T17:18:09,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:09,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48068 deadline: 1733246349219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:09,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:09,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:09,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:09,253 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/ca629c3ec3a14522862ea2f5a863d087 2024-12-03T17:18:09,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/a1ccb970015941188cd39a7d905fabf9 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/a1ccb970015941188cd39a7d905fabf9 2024-12-03T17:18:09,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-03T17:18:09,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/a1ccb970015941188cd39a7d905fabf9, entries=200, sequenceid=383, filesize=14.4 K 2024-12-03T17:18:09,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/f7654c94e6544e30b5652f03efd02ef8 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/f7654c94e6544e30b5652f03efd02ef8 2024-12-03T17:18:09,292 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/f7654c94e6544e30b5652f03efd02ef8, entries=150, sequenceid=383, filesize=12.0 K 2024-12-03T17:18:09,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/ca629c3ec3a14522862ea2f5a863d087 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/ca629c3ec3a14522862ea2f5a863d087 2024-12-03T17:18:09,315 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/ca629c3ec3a14522862ea2f5a863d087, entries=150, sequenceid=383, filesize=12.0 K 2024-12-03T17:18:09,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:09,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246349294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:09,316 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:09,316 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:09,316 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:09,316 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~64.85 KB/66410, heapSize ~178.75 KB/183040, currentSize=127.47 KB/130530 for 68fbb73dc00ff4d04c231a23e30e6f1a in 515ms, sequenceid=383, compaction requested=false 2024-12-03T17:18:09,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 68fbb73dc00ff4d04c231a23e30e6f1a: 2024-12-03T17:18:09,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 68fbb73dc00ff4d04c231a23e30e6f1a 2024-12-03T17:18:09,319 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 68fbb73dc00ff4d04c231a23e30e6f1a 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-12-03T17:18:09,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=A 2024-12-03T17:18:09,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:18:09,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=B 2024-12-03T17:18:09,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:18:09,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=C 2024-12-03T17:18:09,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:18:09,328 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-03T17:18:09,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:09,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. as already flushing 2024-12-03T17:18:09,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:09,329 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:09,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:09,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:09,341 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/f9b0793a6f5d459e8a2f0bb4ace9879e is 50, key is test_row_0/A:col10/1733246288945/Put/seqid=0 2024-12-03T17:18:09,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742331_1507 (size=14741) 2024-12-03T17:18:09,384 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/f9b0793a6f5d459e8a2f0bb4ace9879e 2024-12-03T17:18:09,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/abae311f7dd744cf811412be7b63bf6f is 50, key is test_row_0/B:col10/1733246288945/Put/seqid=0 2024-12-03T17:18:09,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:09,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246349374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:09,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:09,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246349376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:09,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:09,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:09,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:09,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:09,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:09,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:09,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:09,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246349378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:09,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:09,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:09,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:09,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742332_1508 (size=12301) 2024-12-03T17:18:09,419 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/abae311f7dd744cf811412be7b63bf6f 2024-12-03T17:18:09,427 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/3666936f953d4658920b9de8634f5256 is 50, key is test_row_0/C:col10/1733246288945/Put/seqid=0 2024-12-03T17:18:09,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742333_1509 (size=12301) 2024-12-03T17:18:09,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-03T17:18:09,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-03T17:18:09,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:09,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. as already flushing 2024-12-03T17:18:09,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:09,489 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:09,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:09,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:09,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:09,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246349514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:09,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:09,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246349514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:09,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:09,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246349514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:09,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:09,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:09,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:09,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:09,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:09,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:09,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:09,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:09,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:09,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:09,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48068 deadline: 1733246349544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:09,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:09,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:09,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:09,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:09,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246349624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:09,642 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:09,642 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:09,642 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:09,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-03T17:18:09,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:09,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. as already flushing 2024-12-03T17:18:09,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:09,650 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:09,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:09,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:09,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:09,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246349734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:09,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:09,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:09,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:09,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:09,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246349734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:09,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:09,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246349735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:09,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:09,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:09,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:09,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:09,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:09,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:09,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-03T17:18:09,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-03T17:18:09,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:09,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. as already flushing 2024-12-03T17:18:09,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:09,805 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:09,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:09,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:09,874 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/3666936f953d4658920b9de8634f5256 2024-12-03T17:18:09,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/f9b0793a6f5d459e8a2f0bb4ace9879e as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/f9b0793a6f5d459e8a2f0bb4ace9879e 2024-12-03T17:18:09,912 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/f9b0793a6f5d459e8a2f0bb4ace9879e, entries=200, sequenceid=417, filesize=14.4 K 2024-12-03T17:18:09,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/abae311f7dd744cf811412be7b63bf6f as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/abae311f7dd744cf811412be7b63bf6f 2024-12-03T17:18:09,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/abae311f7dd744cf811412be7b63bf6f, entries=150, sequenceid=417, filesize=12.0 K 2024-12-03T17:18:09,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/3666936f953d4658920b9de8634f5256 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/3666936f953d4658920b9de8634f5256 2024-12-03T17:18:09,944 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/3666936f953d4658920b9de8634f5256, entries=150, sequenceid=417, filesize=12.0 K 2024-12-03T17:18:09,945 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~370.88 KB/379776, currentSize=53.67 KB/54960 for 68fbb73dc00ff4d04c231a23e30e6f1a in 625ms, sequenceid=417, compaction requested=true 2024-12-03T17:18:09,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 68fbb73dc00ff4d04c231a23e30e6f1a: 2024-12-03T17:18:09,945 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:18:09,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68fbb73dc00ff4d04c231a23e30e6f1a:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T17:18:09,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:18:09,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68fbb73dc00ff4d04c231a23e30e6f1a:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T17:18:09,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:18:09,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68fbb73dc00ff4d04c231a23e30e6f1a:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T17:18:09,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:18:09,945 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:18:09,946 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:18:09,946 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42465 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:18:09,946 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): 68fbb73dc00ff4d04c231a23e30e6f1a/A is initiating minor compaction (all files) 2024-12-03T17:18:09,946 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): 68fbb73dc00ff4d04c231a23e30e6f1a/B is initiating minor compaction (all files) 2024-12-03T17:18:09,946 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68fbb73dc00ff4d04c231a23e30e6f1a/A in TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:09,946 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68fbb73dc00ff4d04c231a23e30e6f1a/B in TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:09,947 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/f1a4a6efba204836ac341121e0dd5a27, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/a1ccb970015941188cd39a7d905fabf9, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/f9b0793a6f5d459e8a2f0bb4ace9879e] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp, totalSize=41.5 K 2024-12-03T17:18:09,947 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/a2f3ced2d96f4f5c8e036f6daa544495, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/f7654c94e6544e30b5652f03efd02ef8, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/abae311f7dd744cf811412be7b63bf6f] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp, totalSize=36.7 K 2024-12-03T17:18:09,947 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2f3ced2d96f4f5c8e036f6daa544495, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1733246287897 2024-12-03T17:18:09,947 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting f1a4a6efba204836ac341121e0dd5a27, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1733246287897 2024-12-03T17:18:09,948 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting a1ccb970015941188cd39a7d905fabf9, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1733246288092 2024-12-03T17:18:09,948 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7654c94e6544e30b5652f03efd02ef8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1733246288099 2024-12-03T17:18:09,948 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting f9b0793a6f5d459e8a2f0bb4ace9879e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1733246288945 2024-12-03T17:18:09,948 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting abae311f7dd744cf811412be7b63bf6f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1733246288945 2024-12-03T17:18:09,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-03T17:18:09,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:09,962 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 68fbb73dc00ff4d04c231a23e30e6f1a 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-03T17:18:09,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=A 2024-12-03T17:18:09,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:09,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=B 2024-12-03T17:18:09,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:09,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=C 2024-12-03T17:18:09,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:09,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/cbc947d731a449ceaad2be030ea5030e is 50, key is test_row_1/A:col10/1733246289369/Put/seqid=0 2024-12-03T17:18:09,985 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68fbb73dc00ff4d04c231a23e30e6f1a#B#compaction#435 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:18:09,985 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/f76fb76cc73a487daba6d7f673f41d68 is 50, key is test_row_0/B:col10/1733246288945/Put/seqid=0 2024-12-03T17:18:10,001 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68fbb73dc00ff4d04c231a23e30e6f1a#A#compaction#436 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:18:10,001 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/12316ae6c5744dd38c5413329d2b9c06 is 50, key is test_row_0/A:col10/1733246288945/Put/seqid=0 2024-12-03T17:18:10,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742335_1511 (size=13085) 2024-12-03T17:18:10,027 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/f76fb76cc73a487daba6d7f673f41d68 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/f76fb76cc73a487daba6d7f673f41d68 2024-12-03T17:18:10,034 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 68fbb73dc00ff4d04c231a23e30e6f1a/B of 68fbb73dc00ff4d04c231a23e30e6f1a into f76fb76cc73a487daba6d7f673f41d68(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:18:10,034 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68fbb73dc00ff4d04c231a23e30e6f1a: 2024-12-03T17:18:10,034 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., storeName=68fbb73dc00ff4d04c231a23e30e6f1a/B, priority=13, startTime=1733246289945; duration=0sec 2024-12-03T17:18:10,034 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:18:10,034 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68fbb73dc00ff4d04c231a23e30e6f1a:B 2024-12-03T17:18:10,034 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:18:10,035 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:18:10,035 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): 68fbb73dc00ff4d04c231a23e30e6f1a/C is initiating minor compaction (all files) 2024-12-03T17:18:10,035 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68fbb73dc00ff4d04c231a23e30e6f1a/C in TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:10,035 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/9f7113faa3b74899b5027b0e70049639, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/ca629c3ec3a14522862ea2f5a863d087, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/3666936f953d4658920b9de8634f5256] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp, totalSize=36.7 K 2024-12-03T17:18:10,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742334_1510 (size=9857) 2024-12-03T17:18:10,036 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f7113faa3b74899b5027b0e70049639, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1733246287897 2024-12-03T17:18:10,036 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca629c3ec3a14522862ea2f5a863d087, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1733246288099 2024-12-03T17:18:10,036 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3666936f953d4658920b9de8634f5256, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1733246288945 2024-12-03T17:18:10,060 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68fbb73dc00ff4d04c231a23e30e6f1a#C#compaction#437 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:18:10,061 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/18b936b0faa542edb5839514610fa832 is 50, key is test_row_0/C:col10/1733246288945/Put/seqid=0 2024-12-03T17:18:10,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. as already flushing 2024-12-03T17:18:10,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 68fbb73dc00ff4d04c231a23e30e6f1a 2024-12-03T17:18:10,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742336_1512 (size=13085) 2024-12-03T17:18:10,084 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/12316ae6c5744dd38c5413329d2b9c06 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/12316ae6c5744dd38c5413329d2b9c06 2024-12-03T17:18:10,090 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 68fbb73dc00ff4d04c231a23e30e6f1a/A of 68fbb73dc00ff4d04c231a23e30e6f1a into 12316ae6c5744dd38c5413329d2b9c06(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:18:10,090 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68fbb73dc00ff4d04c231a23e30e6f1a: 2024-12-03T17:18:10,090 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., storeName=68fbb73dc00ff4d04c231a23e30e6f1a/A, priority=13, startTime=1733246289945; duration=0sec 2024-12-03T17:18:10,090 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:18:10,090 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68fbb73dc00ff4d04c231a23e30e6f1a:A 2024-12-03T17:18:10,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742337_1513 (size=13051) 2024-12-03T17:18:10,107 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/18b936b0faa542edb5839514610fa832 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/18b936b0faa542edb5839514610fa832 2024-12-03T17:18:10,111 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 68fbb73dc00ff4d04c231a23e30e6f1a/C of 68fbb73dc00ff4d04c231a23e30e6f1a into 18b936b0faa542edb5839514610fa832(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:18:10,111 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68fbb73dc00ff4d04c231a23e30e6f1a: 2024-12-03T17:18:10,111 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., storeName=68fbb73dc00ff4d04c231a23e30e6f1a/C, priority=13, startTime=1733246289945; duration=0sec 2024-12-03T17:18:10,111 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:18:10,111 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68fbb73dc00ff4d04c231a23e30e6f1a:C 2024-12-03T17:18:10,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:10,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246350123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:10,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:10,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48068 deadline: 1733246350125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:10,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:10,141 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:10,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:10,141 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:10,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:10,141 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:10,141 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733246290094, max timestamp=1733246290136Num uniques 859; 2024-12-03T17:18:10,141 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T17:18:10,141 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733246290094, max timestamp=1733246290136Num uniques 859; 2024-12-03T17:18:10,141 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T17:18:10,141 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733246290094, max timestamp=1733246290136Num uniques 859; 2024-12-03T17:18:10,141 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T17:18:10,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:10,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246350215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:10,230 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:10,230 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:10,230 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:10,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:10,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246350226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:10,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:10,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:10,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:10,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246350226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:10,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:10,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:10,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:10,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:10,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:10,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246350244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:10,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:10,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:10,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48068 deadline: 1733246350244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:10,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:10,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:10,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:10,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:10,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:10,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-03T17:18:10,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:10,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246350333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:10,347 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:10,347 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:10,347 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:10,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:10,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246350343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:10,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:10,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:10,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:10,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:10,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246350354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:10,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:10,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:10,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:10,433 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/cbc947d731a449ceaad2be030ea5030e 2024-12-03T17:18:10,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/b56239637bd644648f50e222712a66af is 50, key is test_row_1/B:col10/1733246289369/Put/seqid=0 2024-12-03T17:18:10,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742338_1514 (size=9857) 2024-12-03T17:18:10,450 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/b56239637bd644648f50e222712a66af 2024-12-03T17:18:10,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/956cdbbdc37c4ea4820863edfb60ef69 is 50, key is test_row_1/C:col10/1733246289369/Put/seqid=0 2024-12-03T17:18:10,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:10,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246350463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:10,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:10,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48068 deadline: 1733246350463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:10,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:10,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:10,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:10,475 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:10,475 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:10,475 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:10,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742339_1515 (size=9857) 2024-12-03T17:18:10,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:10,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246350554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:10,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:10,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246350564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:10,571 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:10,571 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:10,571 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:10,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:10,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:10,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:10,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:10,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246350586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:10,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:10,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:10,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:10,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:10,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48068 deadline: 1733246350783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:10,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:10,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:10,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:10,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:10,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246350784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:10,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:10,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:10,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:10,877 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/956cdbbdc37c4ea4820863edfb60ef69 2024-12-03T17:18:10,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/cbc947d731a449ceaad2be030ea5030e as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/cbc947d731a449ceaad2be030ea5030e 2024-12-03T17:18:10,887 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/cbc947d731a449ceaad2be030ea5030e, entries=100, sequenceid=428, filesize=9.6 K 2024-12-03T17:18:10,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/b56239637bd644648f50e222712a66af as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/b56239637bd644648f50e222712a66af 2024-12-03T17:18:10,896 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/b56239637bd644648f50e222712a66af, entries=100, sequenceid=428, filesize=9.6 K 2024-12-03T17:18:10,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/956cdbbdc37c4ea4820863edfb60ef69 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/956cdbbdc37c4ea4820863edfb60ef69 2024-12-03T17:18:10,901 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/956cdbbdc37c4ea4820863edfb60ef69, entries=100, sequenceid=428, filesize=9.6 K 2024-12-03T17:18:10,903 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 68fbb73dc00ff4d04c231a23e30e6f1a in 941ms, sequenceid=428, compaction requested=false 2024-12-03T17:18:10,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 68fbb73dc00ff4d04c231a23e30e6f1a: 2024-12-03T17:18:10,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:10,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-12-03T17:18:10,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-12-03T17:18:10,913 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-12-03T17:18:10,913 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7420 sec 2024-12-03T17:18:10,915 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.7500 sec 2024-12-03T17:18:10,926 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 68fbb73dc00ff4d04c231a23e30e6f1a 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-03T17:18:10,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 68fbb73dc00ff4d04c231a23e30e6f1a 2024-12-03T17:18:10,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=A 2024-12-03T17:18:10,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:18:10,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=B 2024-12-03T17:18:10,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:18:10,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=C 2024-12-03T17:18:10,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:18:10,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/cb22500d5fdb4b7da54dbbc422c49b51 is 50, key is test_row_0/A:col10/1733246290908/Put/seqid=0 2024-12-03T17:18:10,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:10,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246350965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:10,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:10,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:10,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:10,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:10,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246350966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:10,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:10,985 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:10,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246350976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:10,985 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:10,985 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:10,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:10,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:10,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:11,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742340_1516 (size=12301) 2024-12-03T17:18:11,019 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/cb22500d5fdb4b7da54dbbc422c49b51 2024-12-03T17:18:11,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/a7ed377330e54428b46faea0723036ba is 50, key is test_row_0/B:col10/1733246290908/Put/seqid=0 2024-12-03T17:18:11,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742341_1517 (size=12301) 2024-12-03T17:18:11,058 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/a7ed377330e54428b46faea0723036ba 2024-12-03T17:18:11,074 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/d4178f365f494373b5b78469bab5cbbb is 50, key is test_row_0/C:col10/1733246290908/Put/seqid=0 2024-12-03T17:18:11,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:11,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246351083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:11,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:11,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:11,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:11,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:11,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246351094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:11,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:11,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246351094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:11,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:11,109 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:11,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:11,109 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:11,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:11,109 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:11,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742342_1518 (size=12301) 2024-12-03T17:18:11,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:11,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246351305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:11,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:11,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:11,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:11,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:11,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48068 deadline: 1733246351306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:11,311 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:11,311 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:11,311 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:11,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-03T17:18:11,324 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:18:11,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:11,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246351306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:11,325 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T17:18:11,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:11,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:11,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:11,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-12-03T17:18:11,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-03T17:18:11,328 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T17:18:11,328 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T17:18:11,329 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T17:18:11,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:11,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246351314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:11,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:11,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:11,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:11,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:11,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246351314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:11,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:11,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:11,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:11,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-03T17:18:11,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-03T17:18:11,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:11,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. as already flushing 2024-12-03T17:18:11,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:11,491 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:11,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:11,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:11,530 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/d4178f365f494373b5b78469bab5cbbb 2024-12-03T17:18:11,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/cb22500d5fdb4b7da54dbbc422c49b51 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/cb22500d5fdb4b7da54dbbc422c49b51 2024-12-03T17:18:11,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/cb22500d5fdb4b7da54dbbc422c49b51, entries=150, sequenceid=468, filesize=12.0 K 2024-12-03T17:18:11,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/a7ed377330e54428b46faea0723036ba as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/a7ed377330e54428b46faea0723036ba 2024-12-03T17:18:11,575 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/a7ed377330e54428b46faea0723036ba, entries=150, sequenceid=468, filesize=12.0 K 2024-12-03T17:18:11,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/d4178f365f494373b5b78469bab5cbbb as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/d4178f365f494373b5b78469bab5cbbb 2024-12-03T17:18:11,581 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/d4178f365f494373b5b78469bab5cbbb, entries=150, sequenceid=468, filesize=12.0 K 2024-12-03T17:18:11,582 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 68fbb73dc00ff4d04c231a23e30e6f1a in 656ms, sequenceid=468, compaction requested=true 2024-12-03T17:18:11,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 68fbb73dc00ff4d04c231a23e30e6f1a: 2024-12-03T17:18:11,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68fbb73dc00ff4d04c231a23e30e6f1a:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T17:18:11,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:18:11,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68fbb73dc00ff4d04c231a23e30e6f1a:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T17:18:11,582 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:18:11,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:18:11,582 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:18:11,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68fbb73dc00ff4d04c231a23e30e6f1a:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T17:18:11,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:18:11,584 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35243 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:18:11,584 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35243 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:18:11,584 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): 68fbb73dc00ff4d04c231a23e30e6f1a/B is initiating minor compaction (all files) 2024-12-03T17:18:11,584 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): 68fbb73dc00ff4d04c231a23e30e6f1a/A is initiating minor compaction (all files) 2024-12-03T17:18:11,584 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68fbb73dc00ff4d04c231a23e30e6f1a/A in TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:11,584 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68fbb73dc00ff4d04c231a23e30e6f1a/B in TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:11,584 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/f76fb76cc73a487daba6d7f673f41d68, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/b56239637bd644648f50e222712a66af, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/a7ed377330e54428b46faea0723036ba] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp, totalSize=34.4 K 2024-12-03T17:18:11,584 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/12316ae6c5744dd38c5413329d2b9c06, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/cbc947d731a449ceaad2be030ea5030e, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/cb22500d5fdb4b7da54dbbc422c49b51] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp, totalSize=34.4 K 2024-12-03T17:18:11,585 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting f76fb76cc73a487daba6d7f673f41d68, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1733246288945 2024-12-03T17:18:11,585 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12316ae6c5744dd38c5413329d2b9c06, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1733246288945 2024-12-03T17:18:11,586 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting b56239637bd644648f50e222712a66af, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1733246289369 2024-12-03T17:18:11,586 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbc947d731a449ceaad2be030ea5030e, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1733246289369 2024-12-03T17:18:11,586 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting a7ed377330e54428b46faea0723036ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1733246290196 2024-12-03T17:18:11,586 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb22500d5fdb4b7da54dbbc422c49b51, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1733246290196 2024-12-03T17:18:11,601 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68fbb73dc00ff4d04c231a23e30e6f1a#B#compaction#443 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:18:11,601 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/62564263f44446d98dcdfb86f8205c91 is 50, key is test_row_0/B:col10/1733246290908/Put/seqid=0 2024-12-03T17:18:11,612 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68fbb73dc00ff4d04c231a23e30e6f1a#A#compaction#444 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:18:11,613 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/997ae8bfad7548e7816540ef20205c9f is 50, key is test_row_0/A:col10/1733246290908/Put/seqid=0 2024-12-03T17:18:11,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742343_1519 (size=13187) 2024-12-03T17:18:11,634 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/62564263f44446d98dcdfb86f8205c91 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/62564263f44446d98dcdfb86f8205c91 2024-12-03T17:18:11,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742344_1520 (size=13187) 2024-12-03T17:18:11,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-03T17:18:11,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-03T17:18:11,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:11,649 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 68fbb73dc00ff4d04c231a23e30e6f1a 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-03T17:18:11,650 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/997ae8bfad7548e7816540ef20205c9f as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/997ae8bfad7548e7816540ef20205c9f 2024-12-03T17:18:11,660 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 68fbb73dc00ff4d04c231a23e30e6f1a/B of 68fbb73dc00ff4d04c231a23e30e6f1a into 62564263f44446d98dcdfb86f8205c91(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:18:11,660 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68fbb73dc00ff4d04c231a23e30e6f1a: 2024-12-03T17:18:11,660 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., storeName=68fbb73dc00ff4d04c231a23e30e6f1a/B, priority=13, startTime=1733246291582; duration=0sec 2024-12-03T17:18:11,660 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 68fbb73dc00ff4d04c231a23e30e6f1a/A of 68fbb73dc00ff4d04c231a23e30e6f1a into 997ae8bfad7548e7816540ef20205c9f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:18:11,660 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68fbb73dc00ff4d04c231a23e30e6f1a: 2024-12-03T17:18:11,660 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., storeName=68fbb73dc00ff4d04c231a23e30e6f1a/A, priority=13, startTime=1733246291582; duration=0sec 2024-12-03T17:18:11,660 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:18:11,660 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68fbb73dc00ff4d04c231a23e30e6f1a:B 2024-12-03T17:18:11,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 68fbb73dc00ff4d04c231a23e30e6f1a 2024-12-03T17:18:11,660 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:18:11,661 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68fbb73dc00ff4d04c231a23e30e6f1a:A 2024-12-03T17:18:11,661 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:18:11,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=A 2024-12-03T17:18:11,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:11,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=B 2024-12-03T17:18:11,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:11,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=C 2024-12-03T17:18:11,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:11,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. as already flushing 2024-12-03T17:18:11,662 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:18:11,662 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): 68fbb73dc00ff4d04c231a23e30e6f1a/C is initiating minor compaction (all files) 2024-12-03T17:18:11,662 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68fbb73dc00ff4d04c231a23e30e6f1a/C in TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:11,662 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/18b936b0faa542edb5839514610fa832, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/956cdbbdc37c4ea4820863edfb60ef69, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/d4178f365f494373b5b78469bab5cbbb] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp, totalSize=34.4 K 2024-12-03T17:18:11,663 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 18b936b0faa542edb5839514610fa832, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1733246288945 2024-12-03T17:18:11,663 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 956cdbbdc37c4ea4820863edfb60ef69, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1733246289369 2024-12-03T17:18:11,664 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting d4178f365f494373b5b78469bab5cbbb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1733246290196 2024-12-03T17:18:11,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/125a0a6402da440a8c36e184fcfd49dd is 50, key is test_row_0/A:col10/1733246291638/Put/seqid=0 2024-12-03T17:18:11,677 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68fbb73dc00ff4d04c231a23e30e6f1a#C#compaction#446 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:18:11,678 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/8ed056972b424473aa4bdb4a0f30036b is 50, key is test_row_0/C:col10/1733246290908/Put/seqid=0 2024-12-03T17:18:11,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742345_1521 (size=12301) 2024-12-03T17:18:11,692 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/125a0a6402da440a8c36e184fcfd49dd 2024-12-03T17:18:11,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742346_1522 (size=13153) 2024-12-03T17:18:11,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/1161decc6e3b475da2cca8670ff2a38a is 50, key is test_row_0/B:col10/1733246291638/Put/seqid=0 2024-12-03T17:18:11,712 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/8ed056972b424473aa4bdb4a0f30036b as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/8ed056972b424473aa4bdb4a0f30036b 2024-12-03T17:18:11,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742347_1523 (size=12301) 2024-12-03T17:18:11,719 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/1161decc6e3b475da2cca8670ff2a38a 2024-12-03T17:18:11,721 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 68fbb73dc00ff4d04c231a23e30e6f1a/C of 68fbb73dc00ff4d04c231a23e30e6f1a into 8ed056972b424473aa4bdb4a0f30036b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:18:11,721 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68fbb73dc00ff4d04c231a23e30e6f1a: 2024-12-03T17:18:11,721 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., storeName=68fbb73dc00ff4d04c231a23e30e6f1a/C, priority=13, startTime=1733246291582; duration=0sec 2024-12-03T17:18:11,721 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:18:11,721 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68fbb73dc00ff4d04c231a23e30e6f1a:C 2024-12-03T17:18:11,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/d408791bbec845c6b32d222f3b350826 is 50, key is test_row_0/C:col10/1733246291638/Put/seqid=0 2024-12-03T17:18:11,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742348_1524 (size=12301) 2024-12-03T17:18:11,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733246291718, max timestamp=1733246291753Num uniques 859; 2024-12-03T17:18:11,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T17:18:11,758 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733246291718, max timestamp=1733246291753Num uniques 859; 2024-12-03T17:18:11,758 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T17:18:11,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733246291718, max timestamp=1733246291753Num uniques 859; 2024-12-03T17:18:11,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T17:18:11,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:11,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246351818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:11,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:11,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:11,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:11,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:11,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246351820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:11,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:11,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:11,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:11,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:11,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246351827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:11,843 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:11,843 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:11,843 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:11,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-03T17:18:11,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:11,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246351945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:11,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:11,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:11,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:11,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246351946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:11,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:11,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:11,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:11,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:11,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:11,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246351957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:11,972 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:11,972 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:11,972 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:12,137 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/d408791bbec845c6b32d222f3b350826 2024-12-03T17:18:12,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/125a0a6402da440a8c36e184fcfd49dd as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/125a0a6402da440a8c36e184fcfd49dd 2024-12-03T17:18:12,153 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/125a0a6402da440a8c36e184fcfd49dd, entries=150, sequenceid=480, filesize=12.0 K 2024-12-03T17:18:12,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/1161decc6e3b475da2cca8670ff2a38a as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/1161decc6e3b475da2cca8670ff2a38a 2024-12-03T17:18:12,182 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/1161decc6e3b475da2cca8670ff2a38a, entries=150, sequenceid=480, filesize=12.0 K 2024-12-03T17:18:12,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/d408791bbec845c6b32d222f3b350826 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/d408791bbec845c6b32d222f3b350826 2024-12-03T17:18:12,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:12,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246352174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:12,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:12,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246352174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:12,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:12,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:12,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:12,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:12,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:12,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:12,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:12,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246352183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:12,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:12,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:12,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:12,199 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/d408791bbec845c6b32d222f3b350826, entries=150, sequenceid=480, filesize=12.0 K 2024-12-03T17:18:12,200 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 68fbb73dc00ff4d04c231a23e30e6f1a in 551ms, sequenceid=480, compaction requested=false 2024-12-03T17:18:12,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 68fbb73dc00ff4d04c231a23e30e6f1a: 2024-12-03T17:18:12,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:12,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-12-03T17:18:12,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-12-03T17:18:12,204 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-12-03T17:18:12,204 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 874 msec 2024-12-03T17:18:12,214 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 885 msec 2024-12-03T17:18:12,343 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 68fbb73dc00ff4d04c231a23e30e6f1a 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-12-03T17:18:12,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 68fbb73dc00ff4d04c231a23e30e6f1a 2024-12-03T17:18:12,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733246291809, max timestamp=1733246292337Num uniques 1068; 2024-12-03T17:18:12,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-03T17:18:12,349 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733246291817, max timestamp=1733246292337Num uniques 1068; 2024-12-03T17:18:12,349 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=6870, suffix heap size=139040,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=150 2024-12-03T17:18:12,352 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733246291817, max timestamp=1733246292337Num uniques 1068; 2024-12-03T17:18:12,352 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=6870, suffix heap size=139040,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=150 2024-12-03T17:18:12,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=A 2024-12-03T17:18:12,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:18:12,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=B 2024-12-03T17:18:12,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:18:12,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=C 2024-12-03T17:18:12,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:18:12,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/09bb54acc12d4a5588174446252a8b3a is 50, key is test_row_0/A:col10/1733246292337/Put/seqid=0 2024-12-03T17:18:12,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742349_1525 (size=12301) 2024-12-03T17:18:12,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-03T17:18:12,465 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:18:12,465 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T17:18:12,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-12-03T17:18:12,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-03T17:18:12,479 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T17:18:12,480 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T17:18:12,480 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T17:18:12,519 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733246292468, max timestamp=1733246292515Num uniques 859; 2024-12-03T17:18:12,519 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T17:18:12,522 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733246292468, max timestamp=1733246292515Num uniques 859; 2024-12-03T17:18:12,522 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T17:18:12,524 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733246292468, max timestamp=1733246292515Num uniques 859; 2024-12-03T17:18:12,524 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T17:18:12,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-03T17:18:12,618 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733246292558, max timestamp=1733246292613Num uniques 1018; 2024-12-03T17:18:12,618 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733246292558, max timestamp=1733246292613Num uniques 1018; 2024-12-03T17:18:12,618 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-12-03T17:18:12,618 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-12-03T17:18:12,618 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733246292558, max timestamp=1733246292613Num uniques 1018; 2024-12-03T17:18:12,620 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-12-03T17:18:12,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-03T17:18:12,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:12,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. as already flushing 2024-12-03T17:18:12,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:12,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:12,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:12,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:12,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733246292698, max timestamp=1733246292736Num uniques 1127; 2024-12-03T17:18:12,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8092, totalHeapSize=18648, min timestamp=1733246292698, max timestamp=1733246292736Num uniques 1127; 2024-12-03T17:18:12,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51602, new segment data size=8092, suffix heap size=145176,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=177 2024-12-03T17:18:12,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=8102, suffix heap size=145176,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=177 2024-12-03T17:18:12,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733246292698, max timestamp=1733246292736Num uniques 1127; 2024-12-03T17:18:12,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=8102, suffix heap size=145176,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=177 2024-12-03T17:18:12,788 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-03T17:18:12,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:12,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. as already flushing 2024-12-03T17:18:12,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:12,789 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:12,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:12,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:12,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-03T17:18:12,811 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=519 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/09bb54acc12d4a5588174446252a8b3a 2024-12-03T17:18:12,821 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/ee3d3d87fb80476ea17eb2bb05737b45 is 50, key is test_row_0/B:col10/1733246292337/Put/seqid=0 2024-12-03T17:18:12,830 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1733246292754, max timestamp=1733246292825Num uniques 1036; 2024-12-03T17:18:12,830 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=13096, suffix heap size=135712,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=286 2024-12-03T17:18:12,831 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1733246292754, max timestamp=1733246292825Num uniques 1036; 2024-12-03T17:18:12,831 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=13096, suffix heap size=135712,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=286 2024-12-03T17:18:12,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742350_1526 (size=12301) 2024-12-03T17:18:12,850 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733246292785, max timestamp=1733246292843Num uniques 1036; 2024-12-03T17:18:12,850 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-12-03T17:18:12,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-03T17:18:12,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:12,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. as already flushing 2024-12-03T17:18:12,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:12,951 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:12,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:12,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:13,018 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733246292922, max timestamp=1733246293009Num uniques 1145; 2024-12-03T17:18:13,019 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=8930, suffix heap size=147048,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=195 2024-12-03T17:18:13,020 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=161, cellSize=7366, totalHeapSize=16984, min timestamp=1733246292922, max timestamp=1733246293011Num uniques 1095; 2024-12-03T17:18:13,020 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733246292922, max timestamp=1733246293000Num uniques 1145; 2024-12-03T17:18:13,020 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50140, new segment data size=7366, suffix heap size=141848,new segment heap size=16984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=161 2024-12-03T17:18:13,020 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=6870, suffix heap size=147048,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=150 2024-12-03T17:18:13,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-03T17:18:13,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:13,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. as already flushing 2024-12-03T17:18:13,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:13,104 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:13,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:13,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:13,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-03T17:18:13,146 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733246293084, max timestamp=1733246293142Num uniques 1054; 2024-12-03T17:18:13,146 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-12-03T17:18:13,150 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733246293084, max timestamp=1733246293142Num uniques 1020; 2024-12-03T17:18:13,151 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46716, new segment data size=7050, suffix heap size=134048,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1020, new segment cells count=154 2024-12-03T17:18:13,157 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733246293084, max timestamp=1733246293142Num uniques 1009; 2024-12-03T17:18:13,157 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46210, new segment data size=7050, suffix heap size=132904,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=154 2024-12-03T17:18:13,243 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.15 KB at sequenceid=519 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/ee3d3d87fb80476ea17eb2bb05737b45 2024-12-03T17:18:13,258 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/c6c7b40a160942c4850788fd7c3ec467 is 50, key is test_row_0/C:col10/1733246292337/Put/seqid=0 2024-12-03T17:18:13,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-03T17:18:13,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:13,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. as already flushing 2024-12-03T17:18:13,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:13,259 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:13,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:13,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1733246293222, max timestamp=1733246293249Num uniques 1013; 2024-12-03T17:18:13,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=9748, suffix heap size=133320,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=213 2024-12-03T17:18:13,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:13,269 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1733246293222, max timestamp=1733246293249Num uniques 1013; 2024-12-03T17:18:13,269 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=9748, suffix heap size=133320,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=213 2024-12-03T17:18:13,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1733246293222, max timestamp=1733246293249Num uniques 1013; 2024-12-03T17:18:13,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=9748, suffix heap size=133320,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=213 2024-12-03T17:18:13,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742351_1527 (size=12301) 2024-12-03T17:18:13,413 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1733246293350, max timestamp=1733246293408Num uniques 1072; 2024-12-03T17:18:13,413 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=10162, suffix heap size=139456,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=222 2024-12-03T17:18:13,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733246293350, max timestamp=1733246293408Num uniques 1072; 2024-12-03T17:18:13,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-12-03T17:18:13,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-03T17:18:13,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:13,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. as already flushing 2024-12-03T17:18:13,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:13,418 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:13,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:13,419 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7864, totalHeapSize=18128, min timestamp=1733246293350, max timestamp=1733246293413Num uniques 1072; 2024-12-03T17:18:13,419 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49084, new segment data size=7864, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-12-03T17:18:13,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:13,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-03T17:18:13,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:13,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. as already flushing 2024-12-03T17:18:13,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:13,573 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:13,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:13,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:13,601 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1733246293519, max timestamp=1733246293596Num uniques 1081; 2024-12-03T17:18:13,601 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=10576, suffix heap size=140392,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=231 2024-12-03T17:18:13,604 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1733246293519, max timestamp=1733246293596Num uniques 1031; 2024-12-03T17:18:13,604 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=10576, suffix heap size=135192,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=231 2024-12-03T17:18:13,607 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1733246293519, max timestamp=1733246293596Num uniques 1031; 2024-12-03T17:18:13,607 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=10576, suffix heap size=135192,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=231 2024-12-03T17:18:13,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-03T17:18:13,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=290, cellSize=13280, totalHeapSize=30400, min timestamp=1733246293643, max timestamp=1733246293669Num uniques 1090; 2024-12-03T17:18:13,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=13280, suffix heap size=141328,new segment heap size=30400 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=290 2024-12-03T17:18:13,675 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=290, cellSize=13280, totalHeapSize=30400, min timestamp=1733246293643, max timestamp=1733246293669Num uniques 1090; 2024-12-03T17:18:13,675 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=13280, suffix heap size=141328,new segment heap size=30400 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=290 2024-12-03T17:18:13,680 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=290, cellSize=13280, totalHeapSize=30400, min timestamp=1733246293643, max timestamp=1733246293669Num uniques 1090; 2024-12-03T17:18:13,680 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=13280, suffix heap size=141328,new segment heap size=30400 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=290 2024-12-03T17:18:13,700 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.15 KB at sequenceid=519 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/c6c7b40a160942c4850788fd7c3ec467 2024-12-03T17:18:13,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/09bb54acc12d4a5588174446252a8b3a as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/09bb54acc12d4a5588174446252a8b3a 2024-12-03T17:18:13,719 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/09bb54acc12d4a5588174446252a8b3a, entries=150, sequenceid=519, filesize=12.0 K 2024-12-03T17:18:13,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/ee3d3d87fb80476ea17eb2bb05737b45 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/ee3d3d87fb80476ea17eb2bb05737b45 2024-12-03T17:18:13,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-03T17:18:13,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:13,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. as already flushing 2024-12-03T17:18:13,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:13,731 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:13,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:13,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:13,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/ee3d3d87fb80476ea17eb2bb05737b45, entries=150, sequenceid=519, filesize=12.0 K 2024-12-03T17:18:13,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/c6c7b40a160942c4850788fd7c3ec467 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/c6c7b40a160942c4850788fd7c3ec467 2024-12-03T17:18:13,747 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/c6c7b40a160942c4850788fd7c3ec467, entries=150, sequenceid=519, filesize=12.0 K 2024-12-03T17:18:13,748 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~27.47 KB/28134, heapSize ~66.77 KB/68368, currentSize=107.34 KB/109920 for 68fbb73dc00ff4d04c231a23e30e6f1a in 1405ms, sequenceid=519, compaction requested=true 2024-12-03T17:18:13,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 68fbb73dc00ff4d04c231a23e30e6f1a: 2024-12-03T17:18:13,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68fbb73dc00ff4d04c231a23e30e6f1a:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T17:18:13,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:18:13,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68fbb73dc00ff4d04c231a23e30e6f1a:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T17:18:13,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T17:18:13,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68fbb73dc00ff4d04c231a23e30e6f1a:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T17:18:13,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-03T17:18:13,749 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:18:13,749 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:18:13,757 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:18:13,757 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:18:13,757 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): 68fbb73dc00ff4d04c231a23e30e6f1a/A is initiating minor compaction (all files) 2024-12-03T17:18:13,757 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): 68fbb73dc00ff4d04c231a23e30e6f1a/C is initiating minor compaction (all files) 2024-12-03T17:18:13,757 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68fbb73dc00ff4d04c231a23e30e6f1a/A in TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:13,757 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68fbb73dc00ff4d04c231a23e30e6f1a/C in TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:13,757 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/997ae8bfad7548e7816540ef20205c9f, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/125a0a6402da440a8c36e184fcfd49dd, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/09bb54acc12d4a5588174446252a8b3a] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp, totalSize=36.9 K 2024-12-03T17:18:13,757 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/8ed056972b424473aa4bdb4a0f30036b, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/d408791bbec845c6b32d222f3b350826, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/c6c7b40a160942c4850788fd7c3ec467] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp, totalSize=36.9 K 2024-12-03T17:18:13,758 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ed056972b424473aa4bdb4a0f30036b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1733246290196 2024-12-03T17:18:13,759 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 997ae8bfad7548e7816540ef20205c9f, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1733246290196 2024-12-03T17:18:13,759 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting d408791bbec845c6b32d222f3b350826, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=480, earliestPutTs=1733246290962 2024-12-03T17:18:13,759 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 125a0a6402da440a8c36e184fcfd49dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=480, earliestPutTs=1733246290962 2024-12-03T17:18:13,759 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting c6c7b40a160942c4850788fd7c3ec467, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=519, earliestPutTs=1733246291817 2024-12-03T17:18:13,760 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09bb54acc12d4a5588174446252a8b3a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=519, earliestPutTs=1733246291817 2024-12-03T17:18:13,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 68fbb73dc00ff4d04c231a23e30e6f1a 2024-12-03T17:18:13,767 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 68fbb73dc00ff4d04c231a23e30e6f1a 3/3 column families, dataSize=114.05 KB heapSize=312.94 KB 2024-12-03T17:18:13,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=A 2024-12-03T17:18:13,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:18:13,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=B 2024-12-03T17:18:13,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:18:13,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=C 2024-12-03T17:18:13,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:18:13,788 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68fbb73dc00ff4d04c231a23e30e6f1a#C#compaction#452 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:18:13,789 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/887bf3d23b2f4c92a512d3b91e234f30 is 50, key is test_row_0/C:col10/1733246292337/Put/seqid=0 2024-12-03T17:18:13,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/fe64385bef0c4bcd80ab4e1cb7c5a2d6 is 50, key is test_row_0/A:col10/1733246293730/Put/seqid=0 2024-12-03T17:18:13,808 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68fbb73dc00ff4d04c231a23e30e6f1a#A#compaction#454 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:18:13,809 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/b860bf7cd87c4882bb4f071e56912d86 is 50, key is test_row_0/A:col10/1733246292337/Put/seqid=0 2024-12-03T17:18:13,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:13,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246353799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:13,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:13,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246353801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:13,828 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:13,828 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:13,828 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:13,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:13,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:13,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:13,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:13,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246353809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:13,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:13,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:13,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:13,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742353_1529 (size=17181) 2024-12-03T17:18:13,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:13,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246353813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:13,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:13,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:13,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:13,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742352_1528 (size=13255) 2024-12-03T17:18:13,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:13,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48068 deadline: 1733246353827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:13,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:13,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:13,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:13,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-03T17:18:13,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:13,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. as already flushing 2024-12-03T17:18:13,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:13,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:13,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:13,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742354_1530 (size=13289) 2024-12-03T17:18:13,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:13,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:13,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246353933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:13,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:13,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:13,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:13,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:13,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246353943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:13,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:13,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246353943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:13,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:13,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:13,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:13,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:13,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:13,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:13,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:13,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246353954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:13,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:13,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:13,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:13,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:14,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48068 deadline: 1733246353984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:14,000 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:14,000 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:14,000 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:14,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-03T17:18:14,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:14,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. as already flushing 2024-12-03T17:18:14,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:14,055 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:14,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:14,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:14,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:14,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246354158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:14,168 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:14,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:14,168 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:14,168 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:14,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246354164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:14,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:14,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:14,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:14,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:14,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246354164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:14,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:14,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:14,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:14,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:14,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246354186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:14,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:14,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:14,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:14,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-03T17:18:14,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:14,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. as already flushing 2024-12-03T17:18:14,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:14,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:14,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:14,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:14,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:14,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48068 deadline: 1733246354204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:14,219 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:14,219 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:14,219 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:14,243 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=705 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/fe64385bef0c4bcd80ab4e1cb7c5a2d6 2024-12-03T17:18:14,260 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/023f874d01e144efa206b87f7141af95 is 50, key is test_row_0/B:col10/1733246293730/Put/seqid=0 2024-12-03T17:18:14,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742355_1531 (size=12301) 2024-12-03T17:18:14,300 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/887bf3d23b2f4c92a512d3b91e234f30 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/887bf3d23b2f4c92a512d3b91e234f30 2024-12-03T17:18:14,331 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/b860bf7cd87c4882bb4f071e56912d86 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/b860bf7cd87c4882bb4f071e56912d86 2024-12-03T17:18:14,354 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 68fbb73dc00ff4d04c231a23e30e6f1a/C of 68fbb73dc00ff4d04c231a23e30e6f1a into 887bf3d23b2f4c92a512d3b91e234f30(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:18:14,354 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68fbb73dc00ff4d04c231a23e30e6f1a: 2024-12-03T17:18:14,354 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., storeName=68fbb73dc00ff4d04c231a23e30e6f1a/C, priority=13, startTime=1733246293748; duration=0sec 2024-12-03T17:18:14,355 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:18:14,355 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68fbb73dc00ff4d04c231a23e30e6f1a:C 2024-12-03T17:18:14,355 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:18:14,358 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:18:14,358 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 68fbb73dc00ff4d04c231a23e30e6f1a/A of 68fbb73dc00ff4d04c231a23e30e6f1a into b860bf7cd87c4882bb4f071e56912d86(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:18:14,358 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68fbb73dc00ff4d04c231a23e30e6f1a: 2024-12-03T17:18:14,358 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): 68fbb73dc00ff4d04c231a23e30e6f1a/B is initiating minor compaction (all files) 2024-12-03T17:18:14,358 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., storeName=68fbb73dc00ff4d04c231a23e30e6f1a/A, priority=13, startTime=1733246293748; duration=0sec 2024-12-03T17:18:14,358 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68fbb73dc00ff4d04c231a23e30e6f1a/B in TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:14,358 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:18:14,358 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68fbb73dc00ff4d04c231a23e30e6f1a:A 2024-12-03T17:18:14,358 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/62564263f44446d98dcdfb86f8205c91, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/1161decc6e3b475da2cca8670ff2a38a, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/ee3d3d87fb80476ea17eb2bb05737b45] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp, totalSize=36.9 K 2024-12-03T17:18:14,359 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 62564263f44446d98dcdfb86f8205c91, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1733246290196 2024-12-03T17:18:14,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-03T17:18:14,367 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 1161decc6e3b475da2cca8670ff2a38a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=480, earliestPutTs=1733246290962 2024-12-03T17:18:14,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:14,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. as already flushing 2024-12-03T17:18:14,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:14,370 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:14,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:14,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:14,375 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting ee3d3d87fb80476ea17eb2bb05737b45, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=519, earliestPutTs=1733246291817 2024-12-03T17:18:14,446 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68fbb73dc00ff4d04c231a23e30e6f1a#B#compaction#456 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:18:14,447 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/d4a59482d1e641149e2e4832430e8d54 is 50, key is test_row_0/B:col10/1733246292337/Put/seqid=0 2024-12-03T17:18:14,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:14,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246354473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:14,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:14,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246354473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:14,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:14,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246354474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:14,492 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:14,492 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:14,492 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:14,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:14,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:14,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:14,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:14,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:14,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:14,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-03T17:18:14,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:14,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. as already flushing 2024-12-03T17:18:14,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:14,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:14,531 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:14,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246354513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:14,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:14,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:14,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:14,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:14,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:14,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742356_1532 (size=13289) 2024-12-03T17:18:14,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:14,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48068 deadline: 1733246354525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:14,542 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:14,542 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:14,542 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:14,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-03T17:18:14,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-03T17:18:14,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:14,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. as already flushing 2024-12-03T17:18:14,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:14,691 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:14,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:14,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:14,698 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=705 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/023f874d01e144efa206b87f7141af95 2024-12-03T17:18:14,720 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/33552286ef1840c3b9cf4f7904c59e38 is 50, key is test_row_0/C:col10/1733246293730/Put/seqid=0 2024-12-03T17:18:14,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742357_1533 (size=12301) 2024-12-03T17:18:14,761 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=705 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/33552286ef1840c3b9cf4f7904c59e38 2024-12-03T17:18:14,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/fe64385bef0c4bcd80ab4e1cb7c5a2d6 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/fe64385bef0c4bcd80ab4e1cb7c5a2d6 2024-12-03T17:18:14,781 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/fe64385bef0c4bcd80ab4e1cb7c5a2d6, entries=250, sequenceid=705, filesize=16.8 K 2024-12-03T17:18:14,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/023f874d01e144efa206b87f7141af95 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/023f874d01e144efa206b87f7141af95 2024-12-03T17:18:14,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/023f874d01e144efa206b87f7141af95, entries=150, sequenceid=705, filesize=12.0 K 2024-12-03T17:18:14,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/33552286ef1840c3b9cf4f7904c59e38 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/33552286ef1840c3b9cf4f7904c59e38 2024-12-03T17:18:14,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/33552286ef1840c3b9cf4f7904c59e38, entries=150, sequenceid=705, filesize=12.0 K 2024-12-03T17:18:14,811 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~312.89 KB/320400, currentSize=67.09 KB/68700 for 68fbb73dc00ff4d04c231a23e30e6f1a in 1044ms, sequenceid=705, compaction requested=false 2024-12-03T17:18:14,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 68fbb73dc00ff4d04c231a23e30e6f1a: 2024-12-03T17:18:14,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-03T17:18:14,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:14,851 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 68fbb73dc00ff4d04c231a23e30e6f1a 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T17:18:14,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=A 2024-12-03T17:18:14,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:14,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=B 2024-12-03T17:18:14,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:14,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=C 2024-12-03T17:18:14,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:14,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/9647bdb41d3549ff909af4f303695719 is 50, key is test_row_0/A:col10/1733246293795/Put/seqid=0 2024-12-03T17:18:14,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742358_1534 (size=12301) 2024-12-03T17:18:14,931 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/d4a59482d1e641149e2e4832430e8d54 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/d4a59482d1e641149e2e4832430e8d54 2024-12-03T17:18:14,938 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 68fbb73dc00ff4d04c231a23e30e6f1a/B of 68fbb73dc00ff4d04c231a23e30e6f1a into d4a59482d1e641149e2e4832430e8d54(size=13.0 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:18:14,938 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68fbb73dc00ff4d04c231a23e30e6f1a: 2024-12-03T17:18:14,938 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., storeName=68fbb73dc00ff4d04c231a23e30e6f1a/B, priority=13, startTime=1733246293748; duration=0sec 2024-12-03T17:18:14,938 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:18:14,938 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68fbb73dc00ff4d04c231a23e30e6f1a:B 2024-12-03T17:18:15,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 68fbb73dc00ff4d04c231a23e30e6f1a 2024-12-03T17:18:15,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. as already flushing 2024-12-03T17:18:15,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:15,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48068 deadline: 1733246355092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:15,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:15,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:15,119 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:15,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:15,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246355098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:15,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:15,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:15,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:15,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:15,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246355111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:15,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:15,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:15,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246355115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:15,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:15,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:15,130 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:15,130 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:15,130 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:15,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:15,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246355118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:15,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:15,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:15,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:15,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:15,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48068 deadline: 1733246355223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:15,240 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:15,240 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:15,240 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:15,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:15,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246355234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:15,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:15,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:15,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:15,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:15,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246355234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:15,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:15,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:15,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:15,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:15,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246355239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:15,251 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:15,251 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:15,251 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:15,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:15,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246355246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:15,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:15,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:15,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:15,327 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=720 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/9647bdb41d3549ff909af4f303695719 2024-12-03T17:18:15,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/53730dc4e4a54537b21839619510cae9 is 50, key is test_row_0/B:col10/1733246293795/Put/seqid=0 2024-12-03T17:18:15,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742359_1535 (size=12301) 2024-12-03T17:18:15,387 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=720 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/53730dc4e4a54537b21839619510cae9 2024-12-03T17:18:15,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/53023db435c84383bee48d3365f33e8b is 50, key is test_row_0/C:col10/1733246293795/Put/seqid=0 2024-12-03T17:18:15,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742360_1536 (size=12301) 2024-12-03T17:18:15,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:15,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48068 deadline: 1733246355444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:15,456 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:15,456 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:15,456 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:15,457 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=720 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/53023db435c84383bee48d3365f33e8b 2024-12-03T17:18:15,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/9647bdb41d3549ff909af4f303695719 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/9647bdb41d3549ff909af4f303695719 2024-12-03T17:18:15,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:15,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48024 deadline: 1733246355455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:15,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:15,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:15,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:15,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:15,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48042 deadline: 1733246355455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:15,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:15,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:15,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:15,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48080 deadline: 1733246355458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:15,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:15,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:15,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:15,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:15,481 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/9647bdb41d3549ff909af4f303695719, entries=150, sequenceid=720, filesize=12.0 K 2024-12-03T17:18:15,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:15,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:48016 deadline: 1733246355471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:15,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 , the old value is region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:15,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:15,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., hostname=27e0f852a164,41167,1733246151593, seqNum=2 because the exception is null or not the one we care about 2024-12-03T17:18:15,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/53730dc4e4a54537b21839619510cae9 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/53730dc4e4a54537b21839619510cae9 2024-12-03T17:18:15,507 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/53730dc4e4a54537b21839619510cae9, entries=150, sequenceid=720, filesize=12.0 K 2024-12-03T17:18:15,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/53023db435c84383bee48d3365f33e8b as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/53023db435c84383bee48d3365f33e8b 2024-12-03T17:18:15,547 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/53023db435c84383bee48d3365f33e8b, entries=150, sequenceid=720, filesize=12.0 K 2024-12-03T17:18:15,554 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 68fbb73dc00ff4d04c231a23e30e6f1a in 703ms, sequenceid=720, compaction requested=true 2024-12-03T17:18:15,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 68fbb73dc00ff4d04c231a23e30e6f1a: 2024-12-03T17:18:15,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:15,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-12-03T17:18:15,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-12-03T17:18:15,571 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-12-03T17:18:15,571 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0860 sec 2024-12-03T17:18:15,574 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 3.1070 sec 2024-12-03T17:18:15,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 68fbb73dc00ff4d04c231a23e30e6f1a 2024-12-03T17:18:15,809 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 68fbb73dc00ff4d04c231a23e30e6f1a 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-03T17:18:15,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733246295083, max timestamp=1733246295785Num uniques 859; 2024-12-03T17:18:15,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-03T17:18:15,813 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733246295083, max timestamp=1733246295785Num uniques 859; 2024-12-03T17:18:15,813 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-03T17:18:15,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=A 2024-12-03T17:18:15,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:18:15,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=B 2024-12-03T17:18:15,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:18:15,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=C 2024-12-03T17:18:15,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:18:15,835 INFO [Thread-1986 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1986. 2024-12-03T17:18:15,835 DEBUG [Thread-1986 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:18:15,835 DEBUG [Thread-1986 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:15,835 DEBUG [Thread-1986 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:15,836 INFO [Thread-1988 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1988. 2024-12-03T17:18:15,836 DEBUG [Thread-1988 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:18:15,836 DEBUG [Thread-1988 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:15,836 DEBUG [Thread-1988 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:15,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/8d271307e2e34e0bafaa0d428d63f41b is 50, key is test_row_0/A:col10/1733246295807/Put/seqid=0 2024-12-03T17:18:15,838 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:18:15,838 INFO [Thread-1997 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1997. 2024-12-03T17:18:15,838 DEBUG [Thread-1997 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:18:15,838 DEBUG [Thread-1997 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:15,839 DEBUG [Thread-1997 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:15,839 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:18:15,841 INFO [Thread-1984 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1984. 2024-12-03T17:18:15,841 DEBUG [Thread-1984 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:18:15,841 DEBUG [Thread-1984 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:15,841 DEBUG [Thread-1984 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:15,841 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:18:15,842 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:18:15,848 INFO [Thread-1993 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1993. 2024-12-03T17:18:15,848 DEBUG [Thread-1993 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:18:15,848 DEBUG [Thread-1993 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:15,848 DEBUG [Thread-1993 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:15,849 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:18:15,856 INFO [Thread-2001 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2001. 2024-12-03T17:18:15,856 DEBUG [Thread-2001 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:18:15,856 DEBUG [Thread-2001 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:15,856 DEBUG [Thread-2001 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:15,857 INFO [Thread-1990 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1990. 2024-12-03T17:18:15,857 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:18:15,857 DEBUG [Thread-1990 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:18:15,857 DEBUG [Thread-1990 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:15,857 DEBUG [Thread-1990 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:15,857 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:18:15,858 INFO [Thread-1982 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1982. 2024-12-03T17:18:15,859 DEBUG [Thread-1982 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:18:15,859 DEBUG [Thread-1982 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:15,859 DEBUG [Thread-1982 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:15,859 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:18:15,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742361_1537 (size=12301) 2024-12-03T17:18:15,867 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=744 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/8d271307e2e34e0bafaa0d428d63f41b 2024-12-03T17:18:15,878 INFO [Thread-1999 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1999. 2024-12-03T17:18:15,878 DEBUG [Thread-1999 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:18:15,878 DEBUG [Thread-1999 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:15,878 DEBUG [Thread-1999 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:15,880 INFO [Thread-1995 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1995. 2024-12-03T17:18:15,880 DEBUG [Thread-1995 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:18:15,880 DEBUG [Thread-1995 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:15,880 DEBUG [Thread-1995 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:15,880 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:18:15,880 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:18:15,905 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/fcf653795e9b47f1969bb1bfa883f522 is 50, key is test_row_0/B:col10/1733246295807/Put/seqid=0 2024-12-03T17:18:15,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742362_1538 (size=12301) 2024-12-03T17:18:16,327 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=744 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/fcf653795e9b47f1969bb1bfa883f522 2024-12-03T17:18:16,348 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/4ae34d357c31478286c2e9bd6b4a0154 is 50, key is test_row_0/C:col10/1733246295807/Put/seqid=0 2024-12-03T17:18:16,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742363_1539 (size=12301) 2024-12-03T17:18:16,389 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=744 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/4ae34d357c31478286c2e9bd6b4a0154 2024-12-03T17:18:16,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/8d271307e2e34e0bafaa0d428d63f41b as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/8d271307e2e34e0bafaa0d428d63f41b 2024-12-03T17:18:16,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/8d271307e2e34e0bafaa0d428d63f41b, entries=150, sequenceid=744, filesize=12.0 K 2024-12-03T17:18:16,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/fcf653795e9b47f1969bb1bfa883f522 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/fcf653795e9b47f1969bb1bfa883f522 2024-12-03T17:18:16,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/fcf653795e9b47f1969bb1bfa883f522, entries=150, sequenceid=744, filesize=12.0 K 2024-12-03T17:18:16,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/4ae34d357c31478286c2e9bd6b4a0154 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/4ae34d357c31478286c2e9bd6b4a0154 2024-12-03T17:18:16,432 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/4ae34d357c31478286c2e9bd6b4a0154, entries=150, sequenceid=744, filesize=12.0 K 2024-12-03T17:18:16,443 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~209.05 KB/214064, currentSize=40.25 KB/41220 for 68fbb73dc00ff4d04c231a23e30e6f1a in 633ms, sequenceid=744, compaction requested=true 2024-12-03T17:18:16,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 68fbb73dc00ff4d04c231a23e30e6f1a: 2024-12-03T17:18:16,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68fbb73dc00ff4d04c231a23e30e6f1a:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T17:18:16,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:18:16,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68fbb73dc00ff4d04c231a23e30e6f1a:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T17:18:16,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:18:16,443 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T17:18:16,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68fbb73dc00ff4d04c231a23e30e6f1a:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T17:18:16,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T17:18:16,446 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T17:18:16,447 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55072 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T17:18:16,447 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): 68fbb73dc00ff4d04c231a23e30e6f1a/A is initiating minor compaction (all files) 2024-12-03T17:18:16,447 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68fbb73dc00ff4d04c231a23e30e6f1a/A in TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:16,448 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/b860bf7cd87c4882bb4f071e56912d86, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/fe64385bef0c4bcd80ab4e1cb7c5a2d6, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/9647bdb41d3549ff909af4f303695719, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/8d271307e2e34e0bafaa0d428d63f41b] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp, totalSize=53.8 K 2024-12-03T17:18:16,451 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting b860bf7cd87c4882bb4f071e56912d86, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=519, earliestPutTs=1733246291817 2024-12-03T17:18:16,453 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T17:18:16,453 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): 68fbb73dc00ff4d04c231a23e30e6f1a/B is initiating minor compaction (all files) 2024-12-03T17:18:16,453 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68fbb73dc00ff4d04c231a23e30e6f1a/B in TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:16,453 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/d4a59482d1e641149e2e4832430e8d54, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/023f874d01e144efa206b87f7141af95, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/53730dc4e4a54537b21839619510cae9, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/fcf653795e9b47f1969bb1bfa883f522] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp, totalSize=49.0 K 2024-12-03T17:18:16,453 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting fe64385bef0c4bcd80ab4e1cb7c5a2d6, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=705, earliestPutTs=1733246293689 2024-12-03T17:18:16,454 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4a59482d1e641149e2e4832430e8d54, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=519, earliestPutTs=1733246291817 2024-12-03T17:18:16,455 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 9647bdb41d3549ff909af4f303695719, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=720, earliestPutTs=1733246293795 2024-12-03T17:18:16,459 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 023f874d01e144efa206b87f7141af95, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=705, earliestPutTs=1733246293730 2024-12-03T17:18:16,459 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d271307e2e34e0bafaa0d428d63f41b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=744, earliestPutTs=1733246295774 2024-12-03T17:18:16,459 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53730dc4e4a54537b21839619510cae9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=720, earliestPutTs=1733246293795 2024-12-03T17:18:16,459 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting fcf653795e9b47f1969bb1bfa883f522, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=744, earliestPutTs=1733246295774 2024-12-03T17:18:16,486 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68fbb73dc00ff4d04c231a23e30e6f1a#A#compaction#464 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:18:16,487 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/a62f9bf0159744478e94b72865e3b301 is 50, key is test_row_0/A:col10/1733246295807/Put/seqid=0 2024-12-03T17:18:16,497 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68fbb73dc00ff4d04c231a23e30e6f1a#B#compaction#465 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:18:16,498 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/a9f2b8fad8454be9882c8576c47cfbb7 is 50, key is test_row_0/B:col10/1733246295807/Put/seqid=0 2024-12-03T17:18:16,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742365_1541 (size=13425) 2024-12-03T17:18:16,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742364_1540 (size=13425) 2024-12-03T17:18:16,515 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/a9f2b8fad8454be9882c8576c47cfbb7 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/a9f2b8fad8454be9882c8576c47cfbb7 2024-12-03T17:18:16,524 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 68fbb73dc00ff4d04c231a23e30e6f1a/B of 68fbb73dc00ff4d04c231a23e30e6f1a into a9f2b8fad8454be9882c8576c47cfbb7(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:18:16,524 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68fbb73dc00ff4d04c231a23e30e6f1a: 2024-12-03T17:18:16,524 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., storeName=68fbb73dc00ff4d04c231a23e30e6f1a/B, priority=12, startTime=1733246296443; duration=0sec 2024-12-03T17:18:16,524 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:18:16,524 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68fbb73dc00ff4d04c231a23e30e6f1a:B 2024-12-03T17:18:16,524 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T17:18:16,532 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50158 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T17:18:16,532 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): 68fbb73dc00ff4d04c231a23e30e6f1a/C is initiating minor compaction (all files) 2024-12-03T17:18:16,533 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68fbb73dc00ff4d04c231a23e30e6f1a/C in TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:16,533 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/887bf3d23b2f4c92a512d3b91e234f30, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/33552286ef1840c3b9cf4f7904c59e38, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/53023db435c84383bee48d3365f33e8b, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/4ae34d357c31478286c2e9bd6b4a0154] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp, totalSize=49.0 K 2024-12-03T17:18:16,537 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 887bf3d23b2f4c92a512d3b91e234f30, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=519, earliestPutTs=1733246291817 2024-12-03T17:18:16,543 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33552286ef1840c3b9cf4f7904c59e38, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=705, earliestPutTs=1733246293730 2024-12-03T17:18:16,547 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53023db435c84383bee48d3365f33e8b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=720, earliestPutTs=1733246293795 2024-12-03T17:18:16,548 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ae34d357c31478286c2e9bd6b4a0154, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=744, earliestPutTs=1733246295774 2024-12-03T17:18:16,573 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68fbb73dc00ff4d04c231a23e30e6f1a#C#compaction#466 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:18:16,574 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/2c248089ddc9461fa6d9d6865e50a815 is 50, key is test_row_0/C:col10/1733246295807/Put/seqid=0 2024-12-03T17:18:16,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742366_1542 (size=13391) 2024-12-03T17:18:16,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-03T17:18:16,657 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:18:16,658 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-03T17:18:16,658 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 139 2024-12-03T17:18:16,658 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 140 2024-12-03T17:18:16,658 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 101 2024-12-03T17:18:16,658 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 128 2024-12-03T17:18:16,658 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 143 2024-12-03T17:18:16,658 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-03T17:18:16,658 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-03T17:18:16,658 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 597 2024-12-03T17:18:16,658 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1789 rows 2024-12-03T17:18:16,658 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 593 2024-12-03T17:18:16,658 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1779 rows 2024-12-03T17:18:16,658 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 587 2024-12-03T17:18:16,658 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1761 rows 2024-12-03T17:18:16,658 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 586 2024-12-03T17:18:16,658 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1758 rows 2024-12-03T17:18:16,658 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 590 2024-12-03T17:18:16,658 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1770 rows 2024-12-03T17:18:16,658 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T17:18:16,658 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T17:18:16,658 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:16,658 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:16,658 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:18:16,659 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-03T17:18:16,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-03T17:18:16,674 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733246296674"}]},"ts":"1733246296674"} 2024-12-03T17:18:16,676 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-03T17:18:16,676 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-03T17:18:16,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-03T17:18:16,683 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T17:18:16,685 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=68fbb73dc00ff4d04c231a23e30e6f1a, UNASSIGN}] 2024-12-03T17:18:16,686 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=68fbb73dc00ff4d04c231a23e30e6f1a, UNASSIGN 2024-12-03T17:18:16,692 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=131 updating hbase:meta row=68fbb73dc00ff4d04c231a23e30e6f1a, regionState=CLOSING, regionLocation=27e0f852a164,41167,1733246151593 2024-12-03T17:18:16,698 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=68fbb73dc00ff4d04c231a23e30e6f1a, UNASSIGN because future has completed 2024-12-03T17:18:16,699 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T17:18:16,699 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; CloseRegionProcedure 68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593}] 2024-12-03T17:18:16,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-03T17:18:16,859 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] handler.UnassignRegionHandler(122): Close 68fbb73dc00ff4d04c231a23e30e6f1a 2024-12-03T17:18:16,859 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T17:18:16,859 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1722): Closing 68fbb73dc00ff4d04c231a23e30e6f1a, disabling compactions & flushes 2024-12-03T17:18:16,859 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1993): waiting for 2 compactions to complete for region TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:16,938 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/a62f9bf0159744478e94b72865e3b301 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/a62f9bf0159744478e94b72865e3b301 2024-12-03T17:18:16,955 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 68fbb73dc00ff4d04c231a23e30e6f1a/A of 68fbb73dc00ff4d04c231a23e30e6f1a into a62f9bf0159744478e94b72865e3b301(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:18:16,955 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68fbb73dc00ff4d04c231a23e30e6f1a: 2024-12-03T17:18:16,955 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., storeName=68fbb73dc00ff4d04c231a23e30e6f1a/A, priority=12, startTime=1733246296443; duration=0sec 2024-12-03T17:18:16,955 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:18:16,955 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68fbb73dc00ff4d04c231a23e30e6f1a:A 2024-12-03T17:18:16,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-03T17:18:17,011 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/2c248089ddc9461fa6d9d6865e50a815 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/2c248089ddc9461fa6d9d6865e50a815 2024-12-03T17:18:17,021 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 68fbb73dc00ff4d04c231a23e30e6f1a/C of 68fbb73dc00ff4d04c231a23e30e6f1a into 2c248089ddc9461fa6d9d6865e50a815(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:18:17,021 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68fbb73dc00ff4d04c231a23e30e6f1a: 2024-12-03T17:18:17,021 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., storeName=68fbb73dc00ff4d04c231a23e30e6f1a/C, priority=12, startTime=1733246296443; duration=0sec 2024-12-03T17:18:17,021 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:17,022 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:17,022 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. after waiting 0 ms 2024-12-03T17:18:17,022 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:17,022 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(2902): Flushing 68fbb73dc00ff4d04c231a23e30e6f1a 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-03T17:18:17,022 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=A 2024-12-03T17:18:17,022 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:17,022 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=B 2024-12-03T17:18:17,022 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:17,022 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68fbb73dc00ff4d04c231a23e30e6f1a, store=C 2024-12-03T17:18:17,022 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:17,023 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:18:17,023 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68fbb73dc00ff4d04c231a23e30e6f1a:C 2024-12-03T17:18:17,028 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/73b1cef83627479b826b4ff5d84a97aa is 50, key is test_row_0/A:col10/1733246295847/Put/seqid=0 2024-12-03T17:18:17,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742367_1543 (size=12301) 2024-12-03T17:18:17,039 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=756 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/73b1cef83627479b826b4ff5d84a97aa 2024-12-03T17:18:17,049 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/3affb72eb0e9486bbf5c1ff6dff286a0 is 50, key is test_row_0/B:col10/1733246295847/Put/seqid=0 2024-12-03T17:18:17,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742368_1544 (size=12301) 2024-12-03T17:18:17,059 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=756 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/3affb72eb0e9486bbf5c1ff6dff286a0 2024-12-03T17:18:17,072 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/13886643c39345bb843e3850b475077b is 50, key is test_row_0/C:col10/1733246295847/Put/seqid=0 2024-12-03T17:18:17,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742369_1545 (size=12301) 2024-12-03T17:18:17,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-03T17:18:17,481 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=756 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/13886643c39345bb843e3850b475077b 2024-12-03T17:18:17,488 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/A/73b1cef83627479b826b4ff5d84a97aa as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/73b1cef83627479b826b4ff5d84a97aa 2024-12-03T17:18:17,504 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/73b1cef83627479b826b4ff5d84a97aa, entries=150, sequenceid=756, filesize=12.0 K 2024-12-03T17:18:17,511 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/B/3affb72eb0e9486bbf5c1ff6dff286a0 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/3affb72eb0e9486bbf5c1ff6dff286a0 2024-12-03T17:18:17,517 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/3affb72eb0e9486bbf5c1ff6dff286a0, entries=150, sequenceid=756, filesize=12.0 K 2024-12-03T17:18:17,518 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/.tmp/C/13886643c39345bb843e3850b475077b as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/13886643c39345bb843e3850b475077b 2024-12-03T17:18:17,529 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/13886643c39345bb843e3850b475077b, entries=150, sequenceid=756, filesize=12.0 K 2024-12-03T17:18:17,535 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=0 B/0 for 68fbb73dc00ff4d04c231a23e30e6f1a in 513ms, sequenceid=756, compaction requested=false 2024-12-03T17:18:17,539 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/afe4274cd3d3408198dbc2c6a7d06367, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/b61da39d6b9b4196bdddfc3105a6ddef, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/ad15a0e79f6d4587a564a22114d76754, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/8b1cb59d5fca4511b900ba8a2a176a36, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/a43860ea6f0b4aedae8081309a099397, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/2dc30052fa0f447f9176bf4bf84b502e, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/4dec81aff97f4669ad73b4b0619c7a0b, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/03c9ddf2ef0c461288201c51a63206a3, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/0522c04bc5fb4b23ac61ef9605e8e7ab, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/1603f971bcf745a59fdb1d5cb915caa4, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/daf905ac94ee4333a3dac1e762d41b0e, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/737cd9c702304a3ea414af02c3397960, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/66a2052056f94daaa1582fc84a1750c3, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/efbc2632ffa94a2ba44d64d0c289e43b, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/0531ed71363d4f25a49b8f55c876b9cf, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/cb207c5b3e4944b2b1f83cac126dd576, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/dd4169ab47c74a1d9064f18d1f81f89e, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/0be604b2c044496b9c0f9b6c2bce3ae5, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/e9a277d43db14f5ca3759debbd83438e, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/99a8270564084775be049cd81f479855, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/f1a4a6efba204836ac341121e0dd5a27, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/a1ccb970015941188cd39a7d905fabf9, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/f9b0793a6f5d459e8a2f0bb4ace9879e, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/12316ae6c5744dd38c5413329d2b9c06, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/cbc947d731a449ceaad2be030ea5030e, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/997ae8bfad7548e7816540ef20205c9f, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/cb22500d5fdb4b7da54dbbc422c49b51, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/125a0a6402da440a8c36e184fcfd49dd, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/b860bf7cd87c4882bb4f071e56912d86, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/09bb54acc12d4a5588174446252a8b3a, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/fe64385bef0c4bcd80ab4e1cb7c5a2d6, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/9647bdb41d3549ff909af4f303695719, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/8d271307e2e34e0bafaa0d428d63f41b] to archive 2024-12-03T17:18:17,541 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T17:18:17,547 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/afe4274cd3d3408198dbc2c6a7d06367 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/afe4274cd3d3408198dbc2c6a7d06367 2024-12-03T17:18:17,550 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/b61da39d6b9b4196bdddfc3105a6ddef to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/b61da39d6b9b4196bdddfc3105a6ddef 2024-12-03T17:18:17,551 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/ad15a0e79f6d4587a564a22114d76754 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/ad15a0e79f6d4587a564a22114d76754 2024-12-03T17:18:17,552 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/8b1cb59d5fca4511b900ba8a2a176a36 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/8b1cb59d5fca4511b900ba8a2a176a36 2024-12-03T17:18:17,553 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/a43860ea6f0b4aedae8081309a099397 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/a43860ea6f0b4aedae8081309a099397 2024-12-03T17:18:17,559 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/2dc30052fa0f447f9176bf4bf84b502e to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/2dc30052fa0f447f9176bf4bf84b502e 2024-12-03T17:18:17,567 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/4dec81aff97f4669ad73b4b0619c7a0b to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/4dec81aff97f4669ad73b4b0619c7a0b 2024-12-03T17:18:17,573 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/03c9ddf2ef0c461288201c51a63206a3 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/03c9ddf2ef0c461288201c51a63206a3 2024-12-03T17:18:17,582 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/0522c04bc5fb4b23ac61ef9605e8e7ab to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/0522c04bc5fb4b23ac61ef9605e8e7ab 2024-12-03T17:18:17,591 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/1603f971bcf745a59fdb1d5cb915caa4 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/1603f971bcf745a59fdb1d5cb915caa4 2024-12-03T17:18:17,598 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/daf905ac94ee4333a3dac1e762d41b0e to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/daf905ac94ee4333a3dac1e762d41b0e 2024-12-03T17:18:17,600 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/737cd9c702304a3ea414af02c3397960 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/737cd9c702304a3ea414af02c3397960 2024-12-03T17:18:17,601 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/66a2052056f94daaa1582fc84a1750c3 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/66a2052056f94daaa1582fc84a1750c3 2024-12-03T17:18:17,602 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/efbc2632ffa94a2ba44d64d0c289e43b to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/efbc2632ffa94a2ba44d64d0c289e43b 2024-12-03T17:18:17,603 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/0531ed71363d4f25a49b8f55c876b9cf to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/0531ed71363d4f25a49b8f55c876b9cf 2024-12-03T17:18:17,605 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/cb207c5b3e4944b2b1f83cac126dd576 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/cb207c5b3e4944b2b1f83cac126dd576 2024-12-03T17:18:17,607 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/dd4169ab47c74a1d9064f18d1f81f89e to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/dd4169ab47c74a1d9064f18d1f81f89e 2024-12-03T17:18:17,608 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/0be604b2c044496b9c0f9b6c2bce3ae5 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/0be604b2c044496b9c0f9b6c2bce3ae5 2024-12-03T17:18:17,610 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/e9a277d43db14f5ca3759debbd83438e to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/e9a277d43db14f5ca3759debbd83438e 2024-12-03T17:18:17,611 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/99a8270564084775be049cd81f479855 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/99a8270564084775be049cd81f479855 2024-12-03T17:18:17,613 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/f1a4a6efba204836ac341121e0dd5a27 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/f1a4a6efba204836ac341121e0dd5a27 2024-12-03T17:18:17,617 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/a1ccb970015941188cd39a7d905fabf9 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/a1ccb970015941188cd39a7d905fabf9 2024-12-03T17:18:17,621 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/f9b0793a6f5d459e8a2f0bb4ace9879e to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/f9b0793a6f5d459e8a2f0bb4ace9879e 2024-12-03T17:18:17,624 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/12316ae6c5744dd38c5413329d2b9c06 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/12316ae6c5744dd38c5413329d2b9c06 2024-12-03T17:18:17,625 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/cbc947d731a449ceaad2be030ea5030e to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/cbc947d731a449ceaad2be030ea5030e 2024-12-03T17:18:17,629 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/997ae8bfad7548e7816540ef20205c9f to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/997ae8bfad7548e7816540ef20205c9f 2024-12-03T17:18:17,631 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/cb22500d5fdb4b7da54dbbc422c49b51 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/cb22500d5fdb4b7da54dbbc422c49b51 2024-12-03T17:18:17,636 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/125a0a6402da440a8c36e184fcfd49dd to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/125a0a6402da440a8c36e184fcfd49dd 2024-12-03T17:18:17,640 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/b860bf7cd87c4882bb4f071e56912d86 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/b860bf7cd87c4882bb4f071e56912d86 2024-12-03T17:18:17,642 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/09bb54acc12d4a5588174446252a8b3a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/09bb54acc12d4a5588174446252a8b3a 2024-12-03T17:18:17,642 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/fe64385bef0c4bcd80ab4e1cb7c5a2d6 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/fe64385bef0c4bcd80ab4e1cb7c5a2d6 2024-12-03T17:18:17,643 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/9647bdb41d3549ff909af4f303695719 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/9647bdb41d3549ff909af4f303695719 2024-12-03T17:18:17,644 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/8d271307e2e34e0bafaa0d428d63f41b to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/8d271307e2e34e0bafaa0d428d63f41b 2024-12-03T17:18:17,646 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/6404fdd485dd48d5998c5fa6387618d5, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/dac2fef6753e45d787fc73ba428c58e8, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/51d6cb44a94e48d9a57bf6099b4d800f, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/54ad9b0c5b2c46aab7a67c87671a4b00, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/c7529491e4934f428d46fe1ffcaa1b40, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/a069caafc2ea4f6ebfe3ba6711a1d6af, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/9de49ad238e04e3eaa3b0672225b9a8b, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/e4bdb5853cbb496c9152fda9a0ac0896, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/e97bfbfb9f7542929a85012dc14740a5, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/a52ac70a46624ab7b6921ba79e784e88, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/1c993d416a9d45f199b8e05d2b6e03fb, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/cc14cce709a74882ad02255e647e959c, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/c26bc509add44f7a9e8041aff36ffcc9, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/dcf868a27a8d4d978384df27bce8da2a, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/3018bc5f2f464461a04a8f58e90d37b8, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/09f0f96b4e844faf9510dd246d1f0480, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/c33f9d22233941c1aa17132a89624ba6, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/bfffba2e0e0a4b85aad65f7891a8300c, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/6a982eda020a4844bc26540a674cfd20, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/a2f3ced2d96f4f5c8e036f6daa544495, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/ec5869825b02403caf4acafdf9cd6f07, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/f7654c94e6544e30b5652f03efd02ef8, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/f76fb76cc73a487daba6d7f673f41d68, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/abae311f7dd744cf811412be7b63bf6f, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/b56239637bd644648f50e222712a66af, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/62564263f44446d98dcdfb86f8205c91, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/a7ed377330e54428b46faea0723036ba, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/1161decc6e3b475da2cca8670ff2a38a, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/d4a59482d1e641149e2e4832430e8d54, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/ee3d3d87fb80476ea17eb2bb05737b45, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/023f874d01e144efa206b87f7141af95, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/53730dc4e4a54537b21839619510cae9, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/fcf653795e9b47f1969bb1bfa883f522] to archive 2024-12-03T17:18:17,646 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T17:18:17,651 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/6404fdd485dd48d5998c5fa6387618d5 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/6404fdd485dd48d5998c5fa6387618d5 2024-12-03T17:18:17,652 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/dac2fef6753e45d787fc73ba428c58e8 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/dac2fef6753e45d787fc73ba428c58e8 2024-12-03T17:18:17,654 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/51d6cb44a94e48d9a57bf6099b4d800f to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/51d6cb44a94e48d9a57bf6099b4d800f 2024-12-03T17:18:17,655 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/54ad9b0c5b2c46aab7a67c87671a4b00 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/54ad9b0c5b2c46aab7a67c87671a4b00 2024-12-03T17:18:17,656 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/c7529491e4934f428d46fe1ffcaa1b40 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/c7529491e4934f428d46fe1ffcaa1b40 2024-12-03T17:18:17,658 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/a069caafc2ea4f6ebfe3ba6711a1d6af to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/a069caafc2ea4f6ebfe3ba6711a1d6af 2024-12-03T17:18:17,662 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/9de49ad238e04e3eaa3b0672225b9a8b to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/9de49ad238e04e3eaa3b0672225b9a8b 2024-12-03T17:18:17,670 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/e4bdb5853cbb496c9152fda9a0ac0896 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/e4bdb5853cbb496c9152fda9a0ac0896 2024-12-03T17:18:17,672 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/e97bfbfb9f7542929a85012dc14740a5 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/e97bfbfb9f7542929a85012dc14740a5 2024-12-03T17:18:17,677 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/a52ac70a46624ab7b6921ba79e784e88 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/a52ac70a46624ab7b6921ba79e784e88 2024-12-03T17:18:17,679 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/1c993d416a9d45f199b8e05d2b6e03fb to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/1c993d416a9d45f199b8e05d2b6e03fb 2024-12-03T17:18:17,681 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/cc14cce709a74882ad02255e647e959c to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/cc14cce709a74882ad02255e647e959c 2024-12-03T17:18:17,685 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/c26bc509add44f7a9e8041aff36ffcc9 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/c26bc509add44f7a9e8041aff36ffcc9 2024-12-03T17:18:17,689 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/dcf868a27a8d4d978384df27bce8da2a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/dcf868a27a8d4d978384df27bce8da2a 2024-12-03T17:18:17,693 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/3018bc5f2f464461a04a8f58e90d37b8 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/3018bc5f2f464461a04a8f58e90d37b8 2024-12-03T17:18:17,698 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/09f0f96b4e844faf9510dd246d1f0480 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/09f0f96b4e844faf9510dd246d1f0480 2024-12-03T17:18:17,704 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/c33f9d22233941c1aa17132a89624ba6 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/c33f9d22233941c1aa17132a89624ba6 2024-12-03T17:18:17,711 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/bfffba2e0e0a4b85aad65f7891a8300c to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/bfffba2e0e0a4b85aad65f7891a8300c 2024-12-03T17:18:17,714 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/6a982eda020a4844bc26540a674cfd20 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/6a982eda020a4844bc26540a674cfd20 2024-12-03T17:18:17,723 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/a2f3ced2d96f4f5c8e036f6daa544495 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/a2f3ced2d96f4f5c8e036f6daa544495 2024-12-03T17:18:17,725 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/ec5869825b02403caf4acafdf9cd6f07 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/ec5869825b02403caf4acafdf9cd6f07 2024-12-03T17:18:17,727 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/f7654c94e6544e30b5652f03efd02ef8 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/f7654c94e6544e30b5652f03efd02ef8 2024-12-03T17:18:17,729 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/f76fb76cc73a487daba6d7f673f41d68 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/f76fb76cc73a487daba6d7f673f41d68 2024-12-03T17:18:17,731 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/abae311f7dd744cf811412be7b63bf6f to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/abae311f7dd744cf811412be7b63bf6f 2024-12-03T17:18:17,732 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/b56239637bd644648f50e222712a66af to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/b56239637bd644648f50e222712a66af 2024-12-03T17:18:17,733 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/62564263f44446d98dcdfb86f8205c91 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/62564263f44446d98dcdfb86f8205c91 2024-12-03T17:18:17,734 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/a7ed377330e54428b46faea0723036ba to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/a7ed377330e54428b46faea0723036ba 2024-12-03T17:18:17,735 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/1161decc6e3b475da2cca8670ff2a38a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/1161decc6e3b475da2cca8670ff2a38a 2024-12-03T17:18:17,737 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/d4a59482d1e641149e2e4832430e8d54 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/d4a59482d1e641149e2e4832430e8d54 2024-12-03T17:18:17,738 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/ee3d3d87fb80476ea17eb2bb05737b45 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/ee3d3d87fb80476ea17eb2bb05737b45 2024-12-03T17:18:17,739 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/023f874d01e144efa206b87f7141af95 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/023f874d01e144efa206b87f7141af95 2024-12-03T17:18:17,741 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/53730dc4e4a54537b21839619510cae9 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/53730dc4e4a54537b21839619510cae9 2024-12-03T17:18:17,743 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/fcf653795e9b47f1969bb1bfa883f522 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/fcf653795e9b47f1969bb1bfa883f522 2024-12-03T17:18:17,747 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/a46e339b04fa4b0d833d529e1f99c8a9, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/1a5aa931bf6048129e090251e04f84d4, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/d2463cb5e46f44de922761ea78fdf381, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/64780699e14e41e7878f63426c25acba, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/eddf120eefb5498d92801fc5ced81a96, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/c6f754999a1f4da0ad194e058cb0953a, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/3c92358e2ffb455b99114564e7ae31a1, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/2f3b1bdea074418680fc54179ecf26e1, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/aba98252b1884ab8bfe6932d7dd26e8a, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/2fcb29ebf7ed4bb6866e4c35d0037016, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/c285ac2ead7d43a8a089d3883775c62b, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/dce5227af3b24d91a84a99607d5807ed, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/5f0769e4cf50452b806883717e384009, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/a2eb993352eb4ae799d5a11059842bd4, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/b27940cb751843248d0d82d5f6f54df0, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/c3da1a61f742433b982e263845249ba4, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/ebc4f629ea6245a78ed2310b2c498a83, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/1810a539ba6842b094258a273a218e43, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/9f7113faa3b74899b5027b0e70049639, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/b2c89ab0fe2744f08e88a5f13cdca7e8, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/ca629c3ec3a14522862ea2f5a863d087, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/18b936b0faa542edb5839514610fa832, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/3666936f953d4658920b9de8634f5256, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/956cdbbdc37c4ea4820863edfb60ef69, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/8ed056972b424473aa4bdb4a0f30036b, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/d4178f365f494373b5b78469bab5cbbb, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/d408791bbec845c6b32d222f3b350826, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/887bf3d23b2f4c92a512d3b91e234f30, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/c6c7b40a160942c4850788fd7c3ec467, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/33552286ef1840c3b9cf4f7904c59e38, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/53023db435c84383bee48d3365f33e8b, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/4ae34d357c31478286c2e9bd6b4a0154] to archive 2024-12-03T17:18:17,748 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T17:18:17,750 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/a46e339b04fa4b0d833d529e1f99c8a9 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/a46e339b04fa4b0d833d529e1f99c8a9 2024-12-03T17:18:17,751 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/1a5aa931bf6048129e090251e04f84d4 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/1a5aa931bf6048129e090251e04f84d4 2024-12-03T17:18:17,757 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/d2463cb5e46f44de922761ea78fdf381 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/d2463cb5e46f44de922761ea78fdf381 2024-12-03T17:18:17,760 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/64780699e14e41e7878f63426c25acba to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/64780699e14e41e7878f63426c25acba 2024-12-03T17:18:17,762 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/eddf120eefb5498d92801fc5ced81a96 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/eddf120eefb5498d92801fc5ced81a96 2024-12-03T17:18:17,763 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/c6f754999a1f4da0ad194e058cb0953a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/c6f754999a1f4da0ad194e058cb0953a 2024-12-03T17:18:17,764 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/3c92358e2ffb455b99114564e7ae31a1 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/3c92358e2ffb455b99114564e7ae31a1 2024-12-03T17:18:17,765 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/2f3b1bdea074418680fc54179ecf26e1 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/2f3b1bdea074418680fc54179ecf26e1 2024-12-03T17:18:17,767 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/aba98252b1884ab8bfe6932d7dd26e8a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/aba98252b1884ab8bfe6932d7dd26e8a 2024-12-03T17:18:17,771 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/2fcb29ebf7ed4bb6866e4c35d0037016 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/2fcb29ebf7ed4bb6866e4c35d0037016 2024-12-03T17:18:17,775 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/c285ac2ead7d43a8a089d3883775c62b to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/c285ac2ead7d43a8a089d3883775c62b 2024-12-03T17:18:17,776 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/dce5227af3b24d91a84a99607d5807ed to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/dce5227af3b24d91a84a99607d5807ed 2024-12-03T17:18:17,777 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/5f0769e4cf50452b806883717e384009 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/5f0769e4cf50452b806883717e384009 2024-12-03T17:18:17,778 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/a2eb993352eb4ae799d5a11059842bd4 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/a2eb993352eb4ae799d5a11059842bd4 2024-12-03T17:18:17,780 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/b27940cb751843248d0d82d5f6f54df0 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/b27940cb751843248d0d82d5f6f54df0 2024-12-03T17:18:17,781 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/c3da1a61f742433b982e263845249ba4 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/c3da1a61f742433b982e263845249ba4 2024-12-03T17:18:17,783 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/ebc4f629ea6245a78ed2310b2c498a83 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/ebc4f629ea6245a78ed2310b2c498a83 2024-12-03T17:18:17,786 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/1810a539ba6842b094258a273a218e43 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/1810a539ba6842b094258a273a218e43 2024-12-03T17:18:17,787 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/9f7113faa3b74899b5027b0e70049639 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/9f7113faa3b74899b5027b0e70049639 2024-12-03T17:18:17,788 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/b2c89ab0fe2744f08e88a5f13cdca7e8 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/b2c89ab0fe2744f08e88a5f13cdca7e8 2024-12-03T17:18:17,791 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/ca629c3ec3a14522862ea2f5a863d087 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/ca629c3ec3a14522862ea2f5a863d087 2024-12-03T17:18:17,793 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/18b936b0faa542edb5839514610fa832 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/18b936b0faa542edb5839514610fa832 2024-12-03T17:18:17,794 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/3666936f953d4658920b9de8634f5256 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/3666936f953d4658920b9de8634f5256 2024-12-03T17:18:17,795 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/956cdbbdc37c4ea4820863edfb60ef69 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/956cdbbdc37c4ea4820863edfb60ef69 2024-12-03T17:18:17,797 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/8ed056972b424473aa4bdb4a0f30036b to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/8ed056972b424473aa4bdb4a0f30036b 2024-12-03T17:18:17,798 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/d4178f365f494373b5b78469bab5cbbb to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/d4178f365f494373b5b78469bab5cbbb 2024-12-03T17:18:17,800 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/d408791bbec845c6b32d222f3b350826 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/d408791bbec845c6b32d222f3b350826 2024-12-03T17:18:17,801 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/887bf3d23b2f4c92a512d3b91e234f30 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/887bf3d23b2f4c92a512d3b91e234f30 2024-12-03T17:18:17,802 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/c6c7b40a160942c4850788fd7c3ec467 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/c6c7b40a160942c4850788fd7c3ec467 2024-12-03T17:18:17,804 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/33552286ef1840c3b9cf4f7904c59e38 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/33552286ef1840c3b9cf4f7904c59e38 2024-12-03T17:18:17,805 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/53023db435c84383bee48d3365f33e8b to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/53023db435c84383bee48d3365f33e8b 2024-12-03T17:18:17,806 DEBUG [StoreCloser-TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/4ae34d357c31478286c2e9bd6b4a0154 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/4ae34d357c31478286c2e9bd6b4a0154 2024-12-03T17:18:17,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-03T17:18:17,816 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/recovered.edits/759.seqid, newMaxSeqId=759, maxSeqId=1 2024-12-03T17:18:17,819 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. 2024-12-03T17:18:17,819 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1676): Region close journal for 68fbb73dc00ff4d04c231a23e30e6f1a: Waiting for close lock at 1733246296859Running coprocessor pre-close hooks at 1733246296859Disabling compacts and flushes for region at 1733246296859Disabling writes for close at 1733246297022 (+163 ms)Obtaining lock to block concurrent updates at 1733246297022Preparing flush snapshotting stores in 68fbb73dc00ff4d04c231a23e30e6f1a at 1733246297022Finished memstore snapshotting TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a., syncing WAL and waiting on mvcc, flushsize=dataSize=41220, getHeapSize=123120, getOffHeapSize=0, getCellsCount=900 at 1733246297022Flushing stores of TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a. at 1733246297023 (+1 ms)Flushing 68fbb73dc00ff4d04c231a23e30e6f1a/A: creating writer at 1733246297023Flushing 68fbb73dc00ff4d04c231a23e30e6f1a/A: appending metadata at 1733246297027 (+4 ms)Flushing 68fbb73dc00ff4d04c231a23e30e6f1a/A: closing flushed file at 1733246297028 (+1 ms)Flushing 68fbb73dc00ff4d04c231a23e30e6f1a/B: creating writer at 1733246297046 (+18 ms)Flushing 68fbb73dc00ff4d04c231a23e30e6f1a/B: appending metadata at 1733246297049 (+3 ms)Flushing 68fbb73dc00ff4d04c231a23e30e6f1a/B: closing flushed file at 1733246297049Flushing 68fbb73dc00ff4d04c231a23e30e6f1a/C: creating writer at 1733246297068 (+19 ms)Flushing 68fbb73dc00ff4d04c231a23e30e6f1a/C: appending metadata at 1733246297071 (+3 ms)Flushing 68fbb73dc00ff4d04c231a23e30e6f1a/C: closing flushed file at 1733246297071Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c0529d1: reopening flushed file at 1733246297487 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a213c46: reopening flushed file at 1733246297504 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@fbeeecc: reopening flushed file at 1733246297517 (+13 ms)Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=0 B/0 for 68fbb73dc00ff4d04c231a23e30e6f1a in 513ms, sequenceid=756, compaction requested=false at 1733246297536 (+19 ms)Writing region close event to WAL at 1733246297807 (+271 ms)Running coprocessor post-close hooks at 1733246297819 (+12 ms)Closed at 1733246297819 2024-12-03T17:18:17,825 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] handler.UnassignRegionHandler(157): Closed 68fbb73dc00ff4d04c231a23e30e6f1a 2024-12-03T17:18:17,827 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=131 updating hbase:meta row=68fbb73dc00ff4d04c231a23e30e6f1a, regionState=CLOSED 2024-12-03T17:18:17,829 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=132, ppid=131, state=RUNNABLE, hasLock=false; CloseRegionProcedure 68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 because future has completed 2024-12-03T17:18:17,832 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-12-03T17:18:17,832 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; CloseRegionProcedure 68fbb73dc00ff4d04c231a23e30e6f1a, server=27e0f852a164,41167,1733246151593 in 1.1310 sec 2024-12-03T17:18:17,835 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=131, resume processing ppid=130 2024-12-03T17:18:17,835 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, ppid=130, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=68fbb73dc00ff4d04c231a23e30e6f1a, UNASSIGN in 1.1470 sec 2024-12-03T17:18:17,837 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-12-03T17:18:17,838 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.1530 sec 2024-12-03T17:18:17,839 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733246297839"}]},"ts":"1733246297839"} 2024-12-03T17:18:17,841 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-03T17:18:17,841 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-03T17:18:17,843 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.1820 sec 2024-12-03T17:18:18,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-03T17:18:18,824 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:18:18,825 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-03T17:18:18,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T17:18:18,827 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=133, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T17:18:18,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-03T17:18:18,829 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=133, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T17:18:18,833 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a 2024-12-03T17:18:18,837 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A, FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B, FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C, FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/recovered.edits] 2024-12-03T17:18:18,843 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/73b1cef83627479b826b4ff5d84a97aa to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/73b1cef83627479b826b4ff5d84a97aa 2024-12-03T17:18:18,844 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/a62f9bf0159744478e94b72865e3b301 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/A/a62f9bf0159744478e94b72865e3b301 2024-12-03T17:18:18,848 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/3affb72eb0e9486bbf5c1ff6dff286a0 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/3affb72eb0e9486bbf5c1ff6dff286a0 2024-12-03T17:18:18,849 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/a9f2b8fad8454be9882c8576c47cfbb7 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/B/a9f2b8fad8454be9882c8576c47cfbb7 2024-12-03T17:18:18,852 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/13886643c39345bb843e3850b475077b to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/13886643c39345bb843e3850b475077b 2024-12-03T17:18:18,858 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/2c248089ddc9461fa6d9d6865e50a815 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/C/2c248089ddc9461fa6d9d6865e50a815 2024-12-03T17:18:18,861 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/recovered.edits/759.seqid to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a/recovered.edits/759.seqid 2024-12-03T17:18:18,861 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/68fbb73dc00ff4d04c231a23e30e6f1a 2024-12-03T17:18:18,862 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-03T17:18:18,865 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=133, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T17:18:18,867 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-03T17:18:18,869 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-03T17:18:18,870 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=133, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T17:18:18,870 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-03T17:18:18,871 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733246298871"}]},"ts":"9223372036854775807"} 2024-12-03T17:18:18,873 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-03T17:18:18,873 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 68fbb73dc00ff4d04c231a23e30e6f1a, NAME => 'TestAcidGuarantees,,1733246274435.68fbb73dc00ff4d04c231a23e30e6f1a.', STARTKEY => '', ENDKEY => ''}] 2024-12-03T17:18:18,874 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-03T17:18:18,874 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733246298874"}]},"ts":"9223372036854775807"} 2024-12-03T17:18:18,875 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-03T17:18:18,876 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=133, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T17:18:18,878 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 51 msec 2024-12-03T17:18:18,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-03T17:18:18,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-03T17:18:18,935 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:18:18,948 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250 (was 247) - Thread LEAK? -, OpenFileDescriptor=449 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1505 (was 1546), ProcessCount=11 (was 11), AvailableMemoryMB=4177 (was 4101) - AvailableMemoryMB LEAK? - 2024-12-03T17:18:18,963 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=1505, ProcessCount=11, AvailableMemoryMB=4175 2024-12-03T17:18:18,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T17:18:18,965 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T17:18:18,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=134, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-03T17:18:18,972 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T17:18:18,972 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:18:18,972 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 134 2024-12-03T17:18:18,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-12-03T17:18:18,973 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T17:18:18,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742370_1546 (size=1061) 2024-12-03T17:18:18,996 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4 2024-12-03T17:18:19,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742371_1547 (size=53) 2024-12-03T17:18:19,028 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T17:18:19,028 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 3e5f45a8f2579c6bf86ccea0cc76d7f9, disabling compactions & flushes 2024-12-03T17:18:19,028 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:19,028 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:19,028 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. after waiting 0 ms 2024-12-03T17:18:19,028 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:19,028 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:19,028 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3e5f45a8f2579c6bf86ccea0cc76d7f9: Waiting for close lock at 1733246299028Disabling compacts and flushes for region at 1733246299028Disabling writes for close at 1733246299028Writing region close event to WAL at 1733246299028Closed at 1733246299028 2024-12-03T17:18:19,030 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T17:18:19,030 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733246299030"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733246299030"}]},"ts":"1733246299030"} 2024-12-03T17:18:19,032 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T17:18:19,033 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T17:18:19,033 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733246299033"}]},"ts":"1733246299033"} 2024-12-03T17:18:19,038 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-03T17:18:19,039 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3e5f45a8f2579c6bf86ccea0cc76d7f9, ASSIGN}] 2024-12-03T17:18:19,044 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3e5f45a8f2579c6bf86ccea0cc76d7f9, ASSIGN 2024-12-03T17:18:19,046 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3e5f45a8f2579c6bf86ccea0cc76d7f9, ASSIGN; state=OFFLINE, location=27e0f852a164,41167,1733246151593; forceNewPlan=false, retain=false 2024-12-03T17:18:19,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-12-03T17:18:19,197 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=3e5f45a8f2579c6bf86ccea0cc76d7f9, regionState=OPENING, regionLocation=27e0f852a164,41167,1733246151593 2024-12-03T17:18:19,199 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3e5f45a8f2579c6bf86ccea0cc76d7f9, ASSIGN because future has completed 2024-12-03T17:18:19,200 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593}] 2024-12-03T17:18:19,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-12-03T17:18:19,357 INFO [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:19,358 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(7752): Opening region: {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} 2024-12-03T17:18:19,358 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:19,358 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T17:18:19,358 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(7794): checking encryption for 3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:19,358 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(7797): checking classloading for 3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:19,362 INFO [StoreOpener-3e5f45a8f2579c6bf86ccea0cc76d7f9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:19,364 INFO [StoreOpener-3e5f45a8f2579c6bf86ccea0cc76d7f9-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T17:18:19,364 INFO [StoreOpener-3e5f45a8f2579c6bf86ccea0cc76d7f9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3e5f45a8f2579c6bf86ccea0cc76d7f9 columnFamilyName A 2024-12-03T17:18:19,364 DEBUG [StoreOpener-3e5f45a8f2579c6bf86ccea0cc76d7f9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:18:19,365 INFO [StoreOpener-3e5f45a8f2579c6bf86ccea0cc76d7f9-1 {}] regionserver.HStore(327): Store=3e5f45a8f2579c6bf86ccea0cc76d7f9/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T17:18:19,365 INFO [StoreOpener-3e5f45a8f2579c6bf86ccea0cc76d7f9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:19,366 INFO [StoreOpener-3e5f45a8f2579c6bf86ccea0cc76d7f9-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T17:18:19,366 INFO [StoreOpener-3e5f45a8f2579c6bf86ccea0cc76d7f9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3e5f45a8f2579c6bf86ccea0cc76d7f9 columnFamilyName B 2024-12-03T17:18:19,366 DEBUG [StoreOpener-3e5f45a8f2579c6bf86ccea0cc76d7f9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:18:19,367 INFO [StoreOpener-3e5f45a8f2579c6bf86ccea0cc76d7f9-1 {}] regionserver.HStore(327): Store=3e5f45a8f2579c6bf86ccea0cc76d7f9/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T17:18:19,367 INFO [StoreOpener-3e5f45a8f2579c6bf86ccea0cc76d7f9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:19,368 INFO [StoreOpener-3e5f45a8f2579c6bf86ccea0cc76d7f9-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T17:18:19,368 INFO [StoreOpener-3e5f45a8f2579c6bf86ccea0cc76d7f9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3e5f45a8f2579c6bf86ccea0cc76d7f9 columnFamilyName C 2024-12-03T17:18:19,368 DEBUG [StoreOpener-3e5f45a8f2579c6bf86ccea0cc76d7f9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:18:19,375 INFO [StoreOpener-3e5f45a8f2579c6bf86ccea0cc76d7f9-1 {}] regionserver.HStore(327): Store=3e5f45a8f2579c6bf86ccea0cc76d7f9/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T17:18:19,375 INFO [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:19,375 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1038): replaying wal for 3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:19,376 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:19,379 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:19,382 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1048): stopping wal replay for 3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:19,382 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1060): Cleaning up temporary data for 3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:19,383 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T17:18:19,384 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1093): writing seq id for 3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:19,386 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T17:18:19,387 INFO [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1114): Opened 3e5f45a8f2579c6bf86ccea0cc76d7f9; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61981750, jitterRate=-0.07639995217323303}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T17:18:19,387 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:19,388 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1006): Region open journal for 3e5f45a8f2579c6bf86ccea0cc76d7f9: Running coprocessor pre-open hook at 1733246299358Writing region info on filesystem at 1733246299358Initializing all the Stores at 1733246299359 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733246299359Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733246299362 (+3 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733246299362Cleaning up temporary data from old regions at 1733246299382 (+20 ms)Running coprocessor post-open hooks at 1733246299387 (+5 ms)Region opened successfully at 1733246299387 2024-12-03T17:18:19,389 INFO [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., pid=136, masterSystemTime=1733246299352 2024-12-03T17:18:19,391 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:19,391 INFO [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:19,392 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=3e5f45a8f2579c6bf86ccea0cc76d7f9, regionState=OPEN, openSeqNum=2, regionLocation=27e0f852a164,41167,1733246151593 2024-12-03T17:18:19,394 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=136, ppid=135, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 because future has completed 2024-12-03T17:18:19,396 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-12-03T17:18:19,396 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; OpenRegionProcedure 3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 in 194 msec 2024-12-03T17:18:19,399 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=135, resume processing ppid=134 2024-12-03T17:18:19,399 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, ppid=134, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3e5f45a8f2579c6bf86ccea0cc76d7f9, ASSIGN in 358 msec 2024-12-03T17:18:19,400 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T17:18:19,400 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733246299400"}]},"ts":"1733246299400"} 2024-12-03T17:18:19,400 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T17:18:19,402 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-03T17:18:19,404 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T17:18:19,406 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 439 msec 2024-12-03T17:18:19,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-12-03T17:18:19,614 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:18:19,619 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11971b93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:18:19,619 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:18:19,619 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:18:19,620 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:18:19,621 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:18:19,621 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:18:19,622 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65bbf255, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:18:19,622 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:18:19,622 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:18:19,622 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:19,624 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46724, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:18:19,627 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75b63283, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:18:19,627 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:18:19,628 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:18:19,628 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:18:19,631 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34550, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:18:19,635 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T17:18:19,638 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 27e0f852a164,38523,1733246150595 2024-12-03T17:18:19,638 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@23d3d519 2024-12-03T17:18:19,639 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T17:18:19,640 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46728, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T17:18:19,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-03T17:18:19,646 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T17:18:19,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-03T17:18:19,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742372_1548 (size=1097) 2024-12-03T17:18:19,712 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-03T17:18:19,712 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-03T17:18:19,727 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T17:18:19,737 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3e5f45a8f2579c6bf86ccea0cc76d7f9, REOPEN/MOVE}] 2024-12-03T17:18:19,738 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3e5f45a8f2579c6bf86ccea0cc76d7f9, REOPEN/MOVE 2024-12-03T17:18:19,740 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=3e5f45a8f2579c6bf86ccea0cc76d7f9, regionState=CLOSING, regionLocation=27e0f852a164,41167,1733246151593 2024-12-03T17:18:19,744 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3e5f45a8f2579c6bf86ccea0cc76d7f9, REOPEN/MOVE because future has completed 2024-12-03T17:18:19,744 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T17:18:19,744 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593}] 2024-12-03T17:18:19,902 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(122): Close 3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:19,902 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T17:18:19,903 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1722): Closing 3e5f45a8f2579c6bf86ccea0cc76d7f9, disabling compactions & flushes 2024-12-03T17:18:19,903 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:19,903 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:19,903 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. after waiting 0 ms 2024-12-03T17:18:19,903 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:19,915 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-03T17:18:19,915 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:19,916 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1676): Region close journal for 3e5f45a8f2579c6bf86ccea0cc76d7f9: Waiting for close lock at 1733246299902Running coprocessor pre-close hooks at 1733246299903 (+1 ms)Disabling compacts and flushes for region at 1733246299903Disabling writes for close at 1733246299903Writing region close event to WAL at 1733246299903Running coprocessor post-close hooks at 1733246299915 (+12 ms)Closed at 1733246299915 2024-12-03T17:18:19,916 WARN [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionServer(3299): Not adding moved region record: 3e5f45a8f2579c6bf86ccea0cc76d7f9 to self. 2024-12-03T17:18:19,918 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(157): Closed 3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:19,918 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=3e5f45a8f2579c6bf86ccea0cc76d7f9, regionState=CLOSED 2024-12-03T17:18:19,920 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 because future has completed 2024-12-03T17:18:19,922 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-12-03T17:18:19,922 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseRegionProcedure 3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 in 176 msec 2024-12-03T17:18:19,923 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3e5f45a8f2579c6bf86ccea0cc76d7f9, REOPEN/MOVE; state=CLOSED, location=27e0f852a164,41167,1733246151593; forceNewPlan=false, retain=true 2024-12-03T17:18:20,073 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=3e5f45a8f2579c6bf86ccea0cc76d7f9, regionState=OPENING, regionLocation=27e0f852a164,41167,1733246151593 2024-12-03T17:18:20,078 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3e5f45a8f2579c6bf86ccea0cc76d7f9, REOPEN/MOVE because future has completed 2024-12-03T17:18:20,078 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593}] 2024-12-03T17:18:20,241 INFO [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:20,242 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(7752): Opening region: {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} 2024-12-03T17:18:20,242 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:20,242 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T17:18:20,242 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(7794): checking encryption for 3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:20,242 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(7797): checking classloading for 3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:20,244 INFO [StoreOpener-3e5f45a8f2579c6bf86ccea0cc76d7f9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:20,244 INFO [StoreOpener-3e5f45a8f2579c6bf86ccea0cc76d7f9-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T17:18:20,245 INFO [StoreOpener-3e5f45a8f2579c6bf86ccea0cc76d7f9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3e5f45a8f2579c6bf86ccea0cc76d7f9 columnFamilyName A 2024-12-03T17:18:20,250 DEBUG [StoreOpener-3e5f45a8f2579c6bf86ccea0cc76d7f9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:18:20,255 INFO [StoreOpener-3e5f45a8f2579c6bf86ccea0cc76d7f9-1 {}] regionserver.HStore(327): Store=3e5f45a8f2579c6bf86ccea0cc76d7f9/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T17:18:20,255 INFO [StoreOpener-3e5f45a8f2579c6bf86ccea0cc76d7f9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:20,256 INFO [StoreOpener-3e5f45a8f2579c6bf86ccea0cc76d7f9-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T17:18:20,256 INFO [StoreOpener-3e5f45a8f2579c6bf86ccea0cc76d7f9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3e5f45a8f2579c6bf86ccea0cc76d7f9 columnFamilyName B 2024-12-03T17:18:20,256 DEBUG [StoreOpener-3e5f45a8f2579c6bf86ccea0cc76d7f9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:18:20,258 INFO [StoreOpener-3e5f45a8f2579c6bf86ccea0cc76d7f9-1 {}] regionserver.HStore(327): Store=3e5f45a8f2579c6bf86ccea0cc76d7f9/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T17:18:20,258 INFO [StoreOpener-3e5f45a8f2579c6bf86ccea0cc76d7f9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:20,263 INFO [StoreOpener-3e5f45a8f2579c6bf86ccea0cc76d7f9-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-03T17:18:20,263 INFO [StoreOpener-3e5f45a8f2579c6bf86ccea0cc76d7f9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3e5f45a8f2579c6bf86ccea0cc76d7f9 columnFamilyName C 2024-12-03T17:18:20,264 DEBUG [StoreOpener-3e5f45a8f2579c6bf86ccea0cc76d7f9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:18:20,265 INFO [StoreOpener-3e5f45a8f2579c6bf86ccea0cc76d7f9-1 {}] regionserver.HStore(327): Store=3e5f45a8f2579c6bf86ccea0cc76d7f9/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T17:18:20,265 INFO [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:20,265 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1038): replaying wal for 3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:20,269 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:20,270 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:20,271 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1048): stopping wal replay for 3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:20,272 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1060): Cleaning up temporary data for 3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:20,272 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T17:18:20,273 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1093): writing seq id for 3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:20,274 INFO [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1114): Opened 3e5f45a8f2579c6bf86ccea0cc76d7f9; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62889170, jitterRate=-0.06287834048271179}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T17:18:20,274 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:20,275 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1006): Region open journal for 3e5f45a8f2579c6bf86ccea0cc76d7f9: Running coprocessor pre-open hook at 1733246300242Writing region info on filesystem at 1733246300242Initializing all the Stores at 1733246300243 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733246300243Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733246300243Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733246300244 (+1 ms)Cleaning up temporary data from old regions at 1733246300272 (+28 ms)Running coprocessor post-open hooks at 1733246300274 (+2 ms)Region opened successfully at 1733246300275 (+1 ms) 2024-12-03T17:18:20,276 INFO [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., pid=141, masterSystemTime=1733246300238 2024-12-03T17:18:20,279 DEBUG [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:20,279 INFO [RS_OPEN_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:20,280 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=3e5f45a8f2579c6bf86ccea0cc76d7f9, regionState=OPEN, openSeqNum=5, regionLocation=27e0f852a164,41167,1733246151593 2024-12-03T17:18:20,281 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 because future has completed 2024-12-03T17:18:20,285 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=139 2024-12-03T17:18:20,285 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=139, state=SUCCESS, hasLock=false; OpenRegionProcedure 3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 in 204 msec 2024-12-03T17:18:20,287 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-12-03T17:18:20,287 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3e5f45a8f2579c6bf86ccea0cc76d7f9, REOPEN/MOVE in 549 msec 2024-12-03T17:18:20,289 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-12-03T17:18:20,290 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 561 msec 2024-12-03T17:18:20,292 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 643 msec 2024-12-03T17:18:20,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-03T17:18:20,293 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:18:20,294 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ca6c80d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:18:20,294 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:18:20,294 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:18:20,295 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:18:20,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:18:20,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:18:20,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b39a40d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:18:20,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:18:20,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:18:20,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:20,297 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46748, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:18:20,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74c5a0fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:18:20,298 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14afc90b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:18:20,298 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:18:20,298 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:18:20,299 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:18:20,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:18:20,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:18:20,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dcbea43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:18:20,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:18:20,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:18:20,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:20,303 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46770, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:18:20,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a1d2891, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:18:20,304 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cb57557, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:18:20,305 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:18:20,305 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:18:20,305 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:18:20,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:18:20,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:18:20,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64ef6edd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:18:20,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:18:20,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:18:20,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:20,307 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46786, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:18:20,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18a6f872, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:18:20,312 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e0b6ba1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:18:20,312 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:18:20,312 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:18:20,313 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:18:20,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:18:20,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:18:20,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a0d95a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:18:20,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:18:20,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:18:20,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:20,314 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46818, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:18:20,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c5ebf3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:18:20,315 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19bc8537, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:18:20,316 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:18:20,316 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:18:20,316 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:18:20,316 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:18:20,316 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:18:20,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c9e92ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:18:20,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:18:20,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:18:20,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:20,318 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46848, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:18:20,318 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2120d81b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:18:20,321 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fd90259, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:18:20,321 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:18:20,321 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:18:20,322 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:18:20,322 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:18:20,322 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:18:20,323 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2111fb87, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:18:20,323 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:18:20,323 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:18:20,323 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:20,324 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46878, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:18:20,325 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@162cb9be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:18:20,331 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@594d5ccc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:18:20,331 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:18:20,331 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:18:20,334 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:18:20,335 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:18:20,335 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:18:20,335 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29a79592, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:18:20,335 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:18:20,336 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:18:20,336 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:20,337 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46898, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:18:20,338 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5eb5cf0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:18:20,340 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@267cfa21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:18:20,340 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:18:20,340 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:18:20,341 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:18:20,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:18:20,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:18:20,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2092e07b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:18:20,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:18:20,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:18:20,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:20,343 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46922, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:18:20,343 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@520de850, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:18:20,346 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c58aaf1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:18:20,347 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:18:20,347 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:18:20,347 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:18:20,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:18:20,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:18:20,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@121fb6ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:18:20,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:18:20,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:18:20,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:20,351 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46934, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:18:20,352 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c2b93c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:18:20,354 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d9bca83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:18:20,355 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 27e0f852a164,38523,-1 for getting cluster id 2024-12-03T17:18:20,355 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T17:18:20,356 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ab945027-d9b6-4a24-a77e-0e4db28ff7fe' 2024-12-03T17:18:20,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T17:18:20,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ab945027-d9b6-4a24-a77e-0e4db28ff7fe" 2024-12-03T17:18:20,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12a6cc8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:18:20,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [27e0f852a164,38523,-1] 2024-12-03T17:18:20,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T17:18:20,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:20,359 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46946, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T17:18:20,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c188088, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T17:18:20,360 DEBUG [Thread-2446 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:18:20,361 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:18:20,361 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:18:20,362 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34552, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:18:20,366 DEBUG [Thread-2449 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:18:20,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5] 2024-12-03T17:18:20,366 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:18:20,367 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:18:20,368 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T17:18:20,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees 2024-12-03T17:18:20,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-03T17:18:20,369 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T17:18:20,371 DEBUG [Thread-2453 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:18:20,371 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:18:20,371 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34566, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:18:20,371 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:18:20,371 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T17:18:20,372 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T17:18:20,373 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34582, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:18:20,375 DEBUG [Thread-2438 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:18:20,376 DEBUG [Thread-2442 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:18:20,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5] 2024-12-03T17:18:20,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5] 2024-12-03T17:18:20,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:18:20,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:18:20,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:18:20,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:18:20,379 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34584, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:18:20,380 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34586, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:18:20,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5] 2024-12-03T17:18:20,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5] 2024-12-03T17:18:20,383 DEBUG [Thread-2457 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:18:20,383 DEBUG [Thread-2451 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:18:20,383 DEBUG [Thread-2444 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:18:20,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:18:20,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:18:20,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:18:20,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:18:20,386 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34608, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:18:20,386 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34602, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:18:20,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:20,387 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:18:20,387 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:18:20,387 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5] 2024-12-03T17:18:20,387 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5] 2024-12-03T17:18:20,387 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3e5f45a8f2579c6bf86ccea0cc76d7f9 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-03T17:18:20,389 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34620, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:18:20,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e5f45a8f2579c6bf86ccea0cc76d7f9, store=A 2024-12-03T17:18:20,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:20,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e5f45a8f2579c6bf86ccea0cc76d7f9, store=B 2024-12-03T17:18:20,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:20,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e5f45a8f2579c6bf86ccea0cc76d7f9, store=C 2024-12-03T17:18:20,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:20,392 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5] 2024-12-03T17:18:20,396 DEBUG [Thread-2440 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:18:20,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:18:20,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:18:20,401 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34634, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:18:20,403 DEBUG [Thread-2455 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T17:18:20,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=27e0f852a164,41167,1733246151593, seqNum=-1] 2024-12-03T17:18:20,404 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5] 2024-12-03T17:18:20,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T17:18:20,412 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34638, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T17:18:20,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5] 2024-12-03T17:18:20,440 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120395692dfcaef54da2b8fb78ac603eb81c_3e5f45a8f2579c6bf86ccea0cc76d7f9 is 50, key is test_row_0/A:col10/1733246300380/Put/seqid=0 2024-12-03T17:18:20,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-03T17:18:20,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742373_1549 (size=12154) 2024-12-03T17:18:20,499 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733246300392, max timestamp=1733246300496Num uniques 859; 2024-12-03T17:18:20,499 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T17:18:20,500 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733246300392, max timestamp=1733246300496Num uniques 859; 2024-12-03T17:18:20,500 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T17:18:20,500 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733246300392, max timestamp=1733246300496Num uniques 859; 2024-12-03T17:18:20,500 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/27e0f852a164:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-03T17:18:20,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:20,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246360515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:20,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:20,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246360517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:20,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-03T17:18:20,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:20,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:20,527 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:20,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:20,527 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:20,527 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:20,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:20,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. as already flushing 2024-12-03T17:18:20,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:20,528 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:20,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:20,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:20,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:20,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246360523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:20,535 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:20,535 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:20,535 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:20,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:20,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34634 deadline: 1733246360527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:20,537 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:20,537 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:20,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:20,537 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:20,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34552 deadline: 1733246360528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:20,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:20,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:20,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:20,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:20,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246360634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:20,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:20,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:20,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:20,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:20,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246360634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:20,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:20,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:20,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:20,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:20,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34552 deadline: 1733246360643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:20,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:20,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34634 deadline: 1733246360644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:20,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:20,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:20,649 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:20,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:20,649 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:20,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246360645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:20,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:20,649 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:20,649 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:20,649 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:20,649 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:20,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-03T17:18:20,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-03T17:18:20,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:20,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. as already flushing 2024-12-03T17:18:20,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:20,692 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:20,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:20,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:20,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:20,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246360843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:20,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:20,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:20,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:20,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-03T17:18:20,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:20,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. as already flushing 2024-12-03T17:18:20,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:20,851 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:20,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:20,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:20,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:20,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34634 deadline: 1733246360855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:20,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:20,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246360856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:20,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:20,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34552 deadline: 1733246360858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:20,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:20,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246360859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:20,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:20,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:20,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:20,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:20,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:20,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:20,863 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:20,863 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:20,863 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:20,863 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:20,863 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:20,863 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:20,888 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:18:20,896 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120395692dfcaef54da2b8fb78ac603eb81c_3e5f45a8f2579c6bf86ccea0cc76d7f9 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120395692dfcaef54da2b8fb78ac603eb81c_3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:20,897 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/da44c70085c84a4e8f1091f42225b968, store: [table=TestAcidGuarantees family=A region=3e5f45a8f2579c6bf86ccea0cc76d7f9] 2024-12-03T17:18:20,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/da44c70085c84a4e8f1091f42225b968 is 175, key is test_row_0/A:col10/1733246300380/Put/seqid=0 2024-12-03T17:18:20,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742374_1550 (size=30955) 2024-12-03T17:18:20,958 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/da44c70085c84a4e8f1091f42225b968 2024-12-03T17:18:20,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-03T17:18:21,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-03T17:18:21,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:21,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. as already flushing 2024-12-03T17:18:21,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:21,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:21,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:21,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:21,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/06518fcfd6af4b57aff956adb3201ea9 is 50, key is test_row_0/B:col10/1733246300380/Put/seqid=0 2024-12-03T17:18:21,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742375_1551 (size=12001) 2024-12-03T17:18:21,134 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/06518fcfd6af4b57aff956adb3201ea9 2024-12-03T17:18:21,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:21,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246361157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:21,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:21,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:21,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:21,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-03T17:18:21,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:21,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. as already flushing 2024-12-03T17:18:21,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:21,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:21,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:21,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:21,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:21,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34634 deadline: 1733246361174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:21,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:21,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246361174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:21,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:21,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246361175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:21,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:21,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:21,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:21,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:21,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34552 deadline: 1733246361175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:21,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:21,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:21,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:21,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:21,179 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:21,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:21,179 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:21,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:21,179 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:21,217 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/1181b19357214873a9abc1551db1f38e is 50, key is test_row_0/C:col10/1733246300380/Put/seqid=0 2024-12-03T17:18:21,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742376_1552 (size=12001) 2024-12-03T17:18:21,262 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/1181b19357214873a9abc1551db1f38e 2024-12-03T17:18:21,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/da44c70085c84a4e8f1091f42225b968 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/da44c70085c84a4e8f1091f42225b968 2024-12-03T17:18:21,315 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/da44c70085c84a4e8f1091f42225b968, entries=150, sequenceid=16, filesize=30.2 K 2024-12-03T17:18:21,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/06518fcfd6af4b57aff956adb3201ea9 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/06518fcfd6af4b57aff956adb3201ea9 2024-12-03T17:18:21,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-03T17:18:21,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:21,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. as already flushing 2024-12-03T17:18:21,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:21,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:21,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:21,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:21,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/06518fcfd6af4b57aff956adb3201ea9, entries=150, sequenceid=16, filesize=11.7 K 2024-12-03T17:18:21,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/1181b19357214873a9abc1551db1f38e as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/1181b19357214873a9abc1551db1f38e 2024-12-03T17:18:21,359 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/1181b19357214873a9abc1551db1f38e, entries=150, sequenceid=16, filesize=11.7 K 2024-12-03T17:18:21,361 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=127.47 KB/130530 for 3e5f45a8f2579c6bf86ccea0cc76d7f9 in 974ms, sequenceid=16, compaction requested=false 2024-12-03T17:18:21,361 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-03T17:18:21,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3e5f45a8f2579c6bf86ccea0cc76d7f9: 2024-12-03T17:18:21,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-03T17:18:21,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:21,484 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2902): Flushing 3e5f45a8f2579c6bf86ccea0cc76d7f9 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-12-03T17:18:21,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e5f45a8f2579c6bf86ccea0cc76d7f9, store=A 2024-12-03T17:18:21,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:18:21,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e5f45a8f2579c6bf86ccea0cc76d7f9, store=B 2024-12-03T17:18:21,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:18:21,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e5f45a8f2579c6bf86ccea0cc76d7f9, store=C 2024-12-03T17:18:21,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:18:21,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203574cabd537314f709ba6a2aa816f4590_3e5f45a8f2579c6bf86ccea0cc76d7f9 is 50, key is test_row_0/A:col10/1733246300509/Put/seqid=0 2024-12-03T17:18:21,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-03T17:18:21,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742377_1553 (size=12154) 2024-12-03T17:18:21,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:18:21,545 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203574cabd537314f709ba6a2aa816f4590_3e5f45a8f2579c6bf86ccea0cc76d7f9 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203574cabd537314f709ba6a2aa816f4590_3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:21,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/89222cda79534edcb639e27ba5eaaf9c, store: [table=TestAcidGuarantees family=A region=3e5f45a8f2579c6bf86ccea0cc76d7f9] 2024-12-03T17:18:21,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/89222cda79534edcb639e27ba5eaaf9c is 175, key is test_row_0/A:col10/1733246300509/Put/seqid=0 2024-12-03T17:18:21,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742378_1554 (size=30955) 2024-12-03T17:18:21,564 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/89222cda79534edcb639e27ba5eaaf9c 2024-12-03T17:18:21,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/974766641dcc44cd996e916f9c33e016 is 50, key is test_row_0/B:col10/1733246300509/Put/seqid=0 2024-12-03T17:18:21,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742379_1555 (size=12001) 2024-12-03T17:18:21,627 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/974766641dcc44cd996e916f9c33e016 2024-12-03T17:18:21,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/5a0b591b597f452caaadd2c02195e3aa is 50, key is test_row_0/C:col10/1733246300509/Put/seqid=0 2024-12-03T17:18:21,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742380_1556 (size=12001) 2024-12-03T17:18:21,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:21,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. as already flushing 2024-12-03T17:18:21,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:21,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246361694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:21,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:21,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:21,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34634 deadline: 1733246361695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:21,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:21,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:21,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:21,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246361695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:21,704 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:21,704 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:21,704 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:21,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:21,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246361700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:21,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:21,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:21,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:21,706 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:21,706 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:21,706 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:21,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:21,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34552 deadline: 1733246361702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:21,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:21,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:21,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:21,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:21,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246361814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:21,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:21,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246361814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:21,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:21,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:21,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:21,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:21,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34552 deadline: 1733246361815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:21,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:21,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:21,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:21,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:21,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246361817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:21,825 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:21,825 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:21,825 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:21,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:21,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:21,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:22,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:22,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246362033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:22,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:22,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:22,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:22,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:22,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246362033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:22,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:22,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:22,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:22,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:22,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:22,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34552 deadline: 1733246362033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:22,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246362034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:22,040 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:22,040 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:22,040 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:22,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:22,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:22,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:22,060 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/5a0b591b597f452caaadd2c02195e3aa 2024-12-03T17:18:22,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/89222cda79534edcb639e27ba5eaaf9c as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/89222cda79534edcb639e27ba5eaaf9c 2024-12-03T17:18:22,075 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/89222cda79534edcb639e27ba5eaaf9c, entries=150, sequenceid=50, filesize=30.2 K 2024-12-03T17:18:22,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/974766641dcc44cd996e916f9c33e016 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/974766641dcc44cd996e916f9c33e016 2024-12-03T17:18:22,080 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/974766641dcc44cd996e916f9c33e016, entries=150, sequenceid=50, filesize=11.7 K 2024-12-03T17:18:22,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/5a0b591b597f452caaadd2c02195e3aa as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/5a0b591b597f452caaadd2c02195e3aa 2024-12-03T17:18:22,085 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/5a0b591b597f452caaadd2c02195e3aa, entries=150, sequenceid=50, filesize=11.7 K 2024-12-03T17:18:22,086 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for 3e5f45a8f2579c6bf86ccea0cc76d7f9 in 602ms, sequenceid=50, compaction requested=false 2024-12-03T17:18:22,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2603): Flush status journal for 3e5f45a8f2579c6bf86ccea0cc76d7f9: 2024-12-03T17:18:22,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:22,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=143 2024-12-03T17:18:22,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4169): Remote procedure done, pid=143 2024-12-03T17:18:22,090 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-12-03T17:18:22,090 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7160 sec 2024-12-03T17:18:22,092 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees in 1.7230 sec 2024-12-03T17:18:22,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:22,353 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3e5f45a8f2579c6bf86ccea0cc76d7f9 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-03T17:18:22,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e5f45a8f2579c6bf86ccea0cc76d7f9, store=A 2024-12-03T17:18:22,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:22,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e5f45a8f2579c6bf86ccea0cc76d7f9, store=B 2024-12-03T17:18:22,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:22,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e5f45a8f2579c6bf86ccea0cc76d7f9, store=C 2024-12-03T17:18:22,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:22,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412039d116831a10e42ddb7d9597151a4178f_3e5f45a8f2579c6bf86ccea0cc76d7f9 is 50, key is test_row_0/A:col10/1733246301692/Put/seqid=0 2024-12-03T17:18:22,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742381_1557 (size=12154) 2024-12-03T17:18:22,400 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:18:22,404 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412039d116831a10e42ddb7d9597151a4178f_3e5f45a8f2579c6bf86ccea0cc76d7f9 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039d116831a10e42ddb7d9597151a4178f_3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:22,405 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/7dfe05e5b68d4778b7445b91f707380b, store: [table=TestAcidGuarantees family=A region=3e5f45a8f2579c6bf86ccea0cc76d7f9] 2024-12-03T17:18:22,405 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/7dfe05e5b68d4778b7445b91f707380b is 175, key is test_row_0/A:col10/1733246301692/Put/seqid=0 2024-12-03T17:18:22,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:22,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246362403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:22,412 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:22,412 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:22,412 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:22,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:22,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246362410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:22,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:22,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34552 deadline: 1733246362412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:22,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:22,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:22,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:22,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:22,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:22,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:22,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:22,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246362415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:22,419 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:22,419 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:22,419 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:22,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742382_1558 (size=30955) 2024-12-03T17:18:22,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-03T17:18:22,524 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:18:22,527 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T17:18:22,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:22,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246362524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:22,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees 2024-12-03T17:18:22,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:22,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34552 deadline: 1733246362524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:22,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246362525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:22,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246362525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-03T17:18:22,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:22,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:22,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:22,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:22,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:22,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:22,530 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:22,530 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:22,530 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:22,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:22,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:22,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:22,531 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T17:18:22,533 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T17:18:22,533 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T17:18:22,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-03T17:18:22,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-03T17:18:22,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:22,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. as already flushing 2024-12-03T17:18:22,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:22,686 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:22,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:22,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:22,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:22,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34634 deadline: 1733246362726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:22,727 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 27e0f852a164,41167,1733246151593 for 'test_row_0' in 3e5f45a8f2579c6bf86ccea0cc76d7f9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T17:18:22,728 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:22,728 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:22,728 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:22,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246362733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:22,737 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:22,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:22,737 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:22,737 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34552 deadline: 1733246362733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:22,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:22,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:22,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:22,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:22,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246362734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:22,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:22,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:22,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:22,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:22,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246362746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:22,750 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:22,750 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:22,750 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:22,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-03T17:18:22,843 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=64, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/7dfe05e5b68d4778b7445b91f707380b 2024-12-03T17:18:22,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:22,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. as already flushing 2024-12-03T17:18:22,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:22,843 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:22,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:22,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:22,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-03T17:18:22,864 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/016a7d91fc49421b91a3fc68c25d5192 is 50, key is test_row_0/B:col10/1733246301692/Put/seqid=0 2024-12-03T17:18:22,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742383_1559 (size=12001) 2024-12-03T17:18:22,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-03T17:18:22,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:23,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. as already flushing 2024-12-03T17:18:23,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:23,000 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:23,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:23,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:23,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:23,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246363043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:23,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:23,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34552 deadline: 1733246363043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:23,046 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:23,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:23,046 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:23,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:23,046 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:23,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:23,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:23,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246363054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:23,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:23,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:23,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:23,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:23,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246363064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:23,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:23,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:23,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:23,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-03T17:18:23,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-03T17:18:23,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:23,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. as already flushing 2024-12-03T17:18:23,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:23,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:23,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:23,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:23,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-03T17:18:23,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:23,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. as already flushing 2024-12-03T17:18:23,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:23,320 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:23,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:23,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:23,330 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/016a7d91fc49421b91a3fc68c25d5192 2024-12-03T17:18:23,357 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T17:18:23,369 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/492ec8369f5f4eb591b6969aede211b4 is 50, key is test_row_0/C:col10/1733246301692/Put/seqid=0 2024-12-03T17:18:23,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742384_1560 (size=12001) 2024-12-03T17:18:23,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-03T17:18:23,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:23,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. as already flushing 2024-12-03T17:18:23,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:23,474 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:23,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:23,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:23,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:23,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34552 deadline: 1733246363553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:23,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:23,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246363555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:23,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:23,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:23,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:23,558 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:23,558 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:23,558 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:23,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:23,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246363575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:23,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:23,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:23,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:23,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:23,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246363587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:23,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:23,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:23,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:23,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-03T17:18:23,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:23,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. as already flushing 2024-12-03T17:18:23,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:23,631 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:23,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:23,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:23,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-03T17:18:23,788 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-03T17:18:23,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:23,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. as already flushing 2024-12-03T17:18:23,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:23,788 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:23,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:23,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:23,812 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/492ec8369f5f4eb591b6969aede211b4 2024-12-03T17:18:23,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/7dfe05e5b68d4778b7445b91f707380b as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/7dfe05e5b68d4778b7445b91f707380b 2024-12-03T17:18:23,834 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/7dfe05e5b68d4778b7445b91f707380b, entries=150, sequenceid=64, filesize=30.2 K 2024-12-03T17:18:23,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/016a7d91fc49421b91a3fc68c25d5192 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/016a7d91fc49421b91a3fc68c25d5192 2024-12-03T17:18:23,859 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/016a7d91fc49421b91a3fc68c25d5192, entries=150, sequenceid=64, filesize=11.7 K 2024-12-03T17:18:23,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/492ec8369f5f4eb591b6969aede211b4 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/492ec8369f5f4eb591b6969aede211b4 2024-12-03T17:18:23,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/492ec8369f5f4eb591b6969aede211b4, entries=150, sequenceid=64, filesize=11.7 K 2024-12-03T17:18:23,937 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 3e5f45a8f2579c6bf86ccea0cc76d7f9 in 1584ms, sequenceid=64, compaction requested=true 2024-12-03T17:18:23,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3e5f45a8f2579c6bf86ccea0cc76d7f9: 2024-12-03T17:18:23,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3e5f45a8f2579c6bf86ccea0cc76d7f9:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T17:18:23,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:18:23,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3e5f45a8f2579c6bf86ccea0cc76d7f9:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T17:18:23,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T17:18:23,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3e5f45a8f2579c6bf86ccea0cc76d7f9:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T17:18:23,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-03T17:18:23,938 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:18:23,938 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:18:23,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-03T17:18:23,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:23,950 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2902): Flushing 3e5f45a8f2579c6bf86ccea0cc76d7f9 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-03T17:18:23,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e5f45a8f2579c6bf86ccea0cc76d7f9, store=A 2024-12-03T17:18:23,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:23,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e5f45a8f2579c6bf86ccea0cc76d7f9, store=B 2024-12-03T17:18:23,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:23,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e5f45a8f2579c6bf86ccea0cc76d7f9, store=C 2024-12-03T17:18:23,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:23,951 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:18:23,951 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): 3e5f45a8f2579c6bf86ccea0cc76d7f9/A is initiating minor compaction (all files) 2024-12-03T17:18:23,951 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3e5f45a8f2579c6bf86ccea0cc76d7f9/A in TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:23,951 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/da44c70085c84a4e8f1091f42225b968, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/89222cda79534edcb639e27ba5eaaf9c, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/7dfe05e5b68d4778b7445b91f707380b] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp, totalSize=90.7 K 2024-12-03T17:18:23,951 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:23,951 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. files: [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/da44c70085c84a4e8f1091f42225b968, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/89222cda79534edcb639e27ba5eaaf9c, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/7dfe05e5b68d4778b7445b91f707380b] 2024-12-03T17:18:23,953 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting da44c70085c84a4e8f1091f42225b968, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733246300380 2024-12-03T17:18:23,958 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:18:23,958 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): 3e5f45a8f2579c6bf86ccea0cc76d7f9/C is initiating minor compaction (all files) 2024-12-03T17:18:23,959 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3e5f45a8f2579c6bf86ccea0cc76d7f9/C in TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:23,959 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/1181b19357214873a9abc1551db1f38e, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/5a0b591b597f452caaadd2c02195e3aa, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/492ec8369f5f4eb591b6969aede211b4] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp, totalSize=35.2 K 2024-12-03T17:18:23,959 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 89222cda79534edcb639e27ba5eaaf9c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733246300509 2024-12-03T17:18:23,960 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1181b19357214873a9abc1551db1f38e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733246300380 2024-12-03T17:18:23,960 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 7dfe05e5b68d4778b7445b91f707380b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733246301692 2024-12-03T17:18:23,967 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a0b591b597f452caaadd2c02195e3aa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733246300509 2024-12-03T17:18:23,968 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 492ec8369f5f4eb591b6969aede211b4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733246301692 2024-12-03T17:18:23,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203952c49a72c1a434f898f83e90da760fd_3e5f45a8f2579c6bf86ccea0cc76d7f9 is 50, key is test_row_0/A:col10/1733246302410/Put/seqid=0 2024-12-03T17:18:23,989 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3e5f45a8f2579c6bf86ccea0cc76d7f9] 2024-12-03T17:18:23,997 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3e5f45a8f2579c6bf86ccea0cc76d7f9#C#compaction#481 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:18:23,998 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/477419cffdba468c8fda655cb669b4ae is 50, key is test_row_0/C:col10/1733246301692/Put/seqid=0 2024-12-03T17:18:24,003 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412033cd287e6be1c484f8e94f4bb04fad82c_3e5f45a8f2579c6bf86ccea0cc76d7f9 store=[table=TestAcidGuarantees family=A region=3e5f45a8f2579c6bf86ccea0cc76d7f9] 2024-12-03T17:18:24,005 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412033cd287e6be1c484f8e94f4bb04fad82c_3e5f45a8f2579c6bf86ccea0cc76d7f9, store=[table=TestAcidGuarantees family=A region=3e5f45a8f2579c6bf86ccea0cc76d7f9] 2024-12-03T17:18:24,005 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412033cd287e6be1c484f8e94f4bb04fad82c_3e5f45a8f2579c6bf86ccea0cc76d7f9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3e5f45a8f2579c6bf86ccea0cc76d7f9] 2024-12-03T17:18:24,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742385_1561 (size=12154) 2024-12-03T17:18:24,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742387_1563 (size=4469) 2024-12-03T17:18:24,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742386_1562 (size=12104) 2024-12-03T17:18:24,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:18:24,451 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203952c49a72c1a434f898f83e90da760fd_3e5f45a8f2579c6bf86ccea0cc76d7f9 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203952c49a72c1a434f898f83e90da760fd_3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:24,455 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3e5f45a8f2579c6bf86ccea0cc76d7f9#A#compaction#480 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:18:24,456 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/2b11daa95f2c4d318052b7aeafee9bf0 is 175, key is test_row_0/A:col10/1733246301692/Put/seqid=0 2024-12-03T17:18:24,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/18ef0e4c158545fdb10e7a4eb4091db9, store: [table=TestAcidGuarantees family=A region=3e5f45a8f2579c6bf86ccea0cc76d7f9] 2024-12-03T17:18:24,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/18ef0e4c158545fdb10e7a4eb4091db9 is 175, key is test_row_0/A:col10/1733246302410/Put/seqid=0 2024-12-03T17:18:24,492 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/477419cffdba468c8fda655cb669b4ae as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/477419cffdba468c8fda655cb669b4ae 2024-12-03T17:18:24,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742389_1565 (size=31058) 2024-12-03T17:18:24,499 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3e5f45a8f2579c6bf86ccea0cc76d7f9/C of 3e5f45a8f2579c6bf86ccea0cc76d7f9 into 477419cffdba468c8fda655cb669b4ae(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:18:24,499 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3e5f45a8f2579c6bf86ccea0cc76d7f9: 2024-12-03T17:18:24,499 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., storeName=3e5f45a8f2579c6bf86ccea0cc76d7f9/C, priority=13, startTime=1733246303937; duration=0sec 2024-12-03T17:18:24,499 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:18:24,499 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3e5f45a8f2579c6bf86ccea0cc76d7f9:C 2024-12-03T17:18:24,499 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:18:24,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742388_1564 (size=30955) 2024-12-03T17:18:24,502 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=82, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/18ef0e4c158545fdb10e7a4eb4091db9 2024-12-03T17:18:24,503 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:18:24,503 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): 3e5f45a8f2579c6bf86ccea0cc76d7f9/B is initiating minor compaction (all files) 2024-12-03T17:18:24,503 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3e5f45a8f2579c6bf86ccea0cc76d7f9/B in TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:24,503 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/06518fcfd6af4b57aff956adb3201ea9, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/974766641dcc44cd996e916f9c33e016, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/016a7d91fc49421b91a3fc68c25d5192] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp, totalSize=35.2 K 2024-12-03T17:18:24,504 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06518fcfd6af4b57aff956adb3201ea9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733246300380 2024-12-03T17:18:24,504 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 974766641dcc44cd996e916f9c33e016, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733246300509 2024-12-03T17:18:24,505 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 016a7d91fc49421b91a3fc68c25d5192, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733246301692 2024-12-03T17:18:24,517 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3e5f45a8f2579c6bf86ccea0cc76d7f9#B#compaction#482 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:18:24,517 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/8d8873197a5a446ea8d2b378cb432ffd is 50, key is test_row_0/B:col10/1733246301692/Put/seqid=0 2024-12-03T17:18:24,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/cabdec57535a431284b3e54e5766c2cc is 50, key is test_row_0/B:col10/1733246302410/Put/seqid=0 2024-12-03T17:18:24,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742390_1566 (size=12104) 2024-12-03T17:18:24,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. as already flushing 2024-12-03T17:18:24,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:24,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742391_1567 (size=12001) 2024-12-03T17:18:24,582 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/8d8873197a5a446ea8d2b378cb432ffd as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/8d8873197a5a446ea8d2b378cb432ffd 2024-12-03T17:18:24,583 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/cabdec57535a431284b3e54e5766c2cc 2024-12-03T17:18:24,598 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3e5f45a8f2579c6bf86ccea0cc76d7f9/B of 3e5f45a8f2579c6bf86ccea0cc76d7f9 into 8d8873197a5a446ea8d2b378cb432ffd(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:18:24,598 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3e5f45a8f2579c6bf86ccea0cc76d7f9: 2024-12-03T17:18:24,598 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., storeName=3e5f45a8f2579c6bf86ccea0cc76d7f9/B, priority=13, startTime=1733246303937; duration=0sec 2024-12-03T17:18:24,598 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:18:24,598 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3e5f45a8f2579c6bf86ccea0cc76d7f9:B 2024-12-03T17:18:24,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/fe22d51cc264468c91d2e93055a05acf is 50, key is test_row_0/C:col10/1733246302410/Put/seqid=0 2024-12-03T17:18:24,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:24,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246364613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:24,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:24,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:24,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246364615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:24,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:24,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:24,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:24,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34552 deadline: 1733246364616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:24,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:24,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246364623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:24,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:24,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:24,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:24,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:24,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:24,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:24,625 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:24,625 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:24,625 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:24,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742392_1568 (size=12001) 2024-12-03T17:18:24,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-03T17:18:24,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:24,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246364735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:24,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:24,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:24,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:24,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:24,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34552 deadline: 1733246364735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:24,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:24,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:24,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246364735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:24,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:24,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:24,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:24,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246364735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:24,742 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:24,742 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:24,742 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:24,742 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:24,742 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:24,742 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:24,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:24,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34634 deadline: 1733246364743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:24,746 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 27e0f852a164,41167,1733246151593 for 'test_row_0' in 3e5f45a8f2579c6bf86ccea0cc76d7f9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T17:18:24,746 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:24,746 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:24,746 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:24,909 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/2b11daa95f2c4d318052b7aeafee9bf0 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/2b11daa95f2c4d318052b7aeafee9bf0 2024-12-03T17:18:24,916 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3e5f45a8f2579c6bf86ccea0cc76d7f9/A of 3e5f45a8f2579c6bf86ccea0cc76d7f9 into 2b11daa95f2c4d318052b7aeafee9bf0(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:18:24,916 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3e5f45a8f2579c6bf86ccea0cc76d7f9: 2024-12-03T17:18:24,916 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., storeName=3e5f45a8f2579c6bf86ccea0cc76d7f9/A, priority=13, startTime=1733246303937; duration=0sec 2024-12-03T17:18:24,916 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:18:24,916 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3e5f45a8f2579c6bf86ccea0cc76d7f9:A 2024-12-03T17:18:24,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:24,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34552 deadline: 1733246364944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:24,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:24,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:24,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:24,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:24,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246364945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:24,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:24,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:24,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:24,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:24,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246364955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:24,956 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:24,956 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:24,956 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:24,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:24,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246364958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:24,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:24,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:24,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:25,036 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/fe22d51cc264468c91d2e93055a05acf 2024-12-03T17:18:25,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/18ef0e4c158545fdb10e7a4eb4091db9 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/18ef0e4c158545fdb10e7a4eb4091db9 2024-12-03T17:18:25,058 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/18ef0e4c158545fdb10e7a4eb4091db9, entries=150, sequenceid=82, filesize=30.2 K 2024-12-03T17:18:25,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/cabdec57535a431284b3e54e5766c2cc as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/cabdec57535a431284b3e54e5766c2cc 2024-12-03T17:18:25,068 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/cabdec57535a431284b3e54e5766c2cc, entries=150, sequenceid=82, filesize=11.7 K 2024-12-03T17:18:25,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/fe22d51cc264468c91d2e93055a05acf as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/fe22d51cc264468c91d2e93055a05acf 2024-12-03T17:18:25,104 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/fe22d51cc264468c91d2e93055a05acf, entries=150, sequenceid=82, filesize=11.7 K 2024-12-03T17:18:25,105 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 3e5f45a8f2579c6bf86ccea0cc76d7f9 in 1156ms, sequenceid=82, compaction requested=false 2024-12-03T17:18:25,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2603): Flush status journal for 3e5f45a8f2579c6bf86ccea0cc76d7f9: 2024-12-03T17:18:25,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:25,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=145 2024-12-03T17:18:25,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4169): Remote procedure done, pid=145 2024-12-03T17:18:25,114 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-12-03T17:18:25,114 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5780 sec 2024-12-03T17:18:25,118 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees in 2.5880 sec 2024-12-03T17:18:25,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:25,262 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3e5f45a8f2579c6bf86ccea0cc76d7f9 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-03T17:18:25,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e5f45a8f2579c6bf86ccea0cc76d7f9, store=A 2024-12-03T17:18:25,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:25,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e5f45a8f2579c6bf86ccea0cc76d7f9, store=B 2024-12-03T17:18:25,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:25,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e5f45a8f2579c6bf86ccea0cc76d7f9, store=C 2024-12-03T17:18:25,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:25,297 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120396d5f2a0f408474a8969b08dc3c55201_3e5f45a8f2579c6bf86ccea0cc76d7f9 is 50, key is test_row_0/A:col10/1733246304613/Put/seqid=0 2024-12-03T17:18:25,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:25,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246365322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:25,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:25,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246365324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:25,335 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:25,335 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:25,335 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:25,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:25,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:25,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:25,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:25,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246365328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:25,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:25,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:25,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:25,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34552 deadline: 1733246365333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:25,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:25,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:25,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:25,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:25,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742393_1569 (size=12154) 2024-12-03T17:18:25,351 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:18:25,377 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120396d5f2a0f408474a8969b08dc3c55201_3e5f45a8f2579c6bf86ccea0cc76d7f9 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120396d5f2a0f408474a8969b08dc3c55201_3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:25,379 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/ad4b5c02c88e481f9dd0a3ed9b222efd, store: [table=TestAcidGuarantees family=A region=3e5f45a8f2579c6bf86ccea0cc76d7f9] 2024-12-03T17:18:25,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/ad4b5c02c88e481f9dd0a3ed9b222efd is 175, key is test_row_0/A:col10/1733246304613/Put/seqid=0 2024-12-03T17:18:25,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742394_1570 (size=30955) 2024-12-03T17:18:25,397 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=101, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/ad4b5c02c88e481f9dd0a3ed9b222efd 2024-12-03T17:18:25,416 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/86b9baa9465d403fbfbc892c018d78dc is 50, key is test_row_0/B:col10/1733246304613/Put/seqid=0 2024-12-03T17:18:25,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:25,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246365444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:25,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:25,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246365444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:25,445 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:25,445 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:25,446 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:25,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:25,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:25,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:25,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:25,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246365447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:25,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:25,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:25,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:25,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:25,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34552 deadline: 1733246365453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:25,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:25,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:25,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:25,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742395_1571 (size=12001) 2024-12-03T17:18:25,471 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/86b9baa9465d403fbfbc892c018d78dc 2024-12-03T17:18:25,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/bacea7f9d98f4e399a7af4c655dfce0d is 50, key is test_row_0/C:col10/1733246304613/Put/seqid=0 2024-12-03T17:18:25,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742396_1572 (size=12001) 2024-12-03T17:18:25,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:25,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:25,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246365654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:25,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246365654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:25,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:25,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:25,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:25,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:25,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:25,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:25,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:25,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34552 deadline: 1733246365663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:25,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:25,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246365663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:25,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:25,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:25,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:25,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:25,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:25,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:25,943 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/bacea7f9d98f4e399a7af4c655dfce0d 2024-12-03T17:18:25,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/ad4b5c02c88e481f9dd0a3ed9b222efd as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/ad4b5c02c88e481f9dd0a3ed9b222efd 2024-12-03T17:18:25,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:25,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246365975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:25,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:25,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246365975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:25,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:25,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34552 deadline: 1733246365975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:25,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:25,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:25,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:25,982 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:25,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:25,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:25,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:25,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246365975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:25,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:25,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:25,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:25,983 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:25,983 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:25,983 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:26,007 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/ad4b5c02c88e481f9dd0a3ed9b222efd, entries=150, sequenceid=101, filesize=30.2 K 2024-12-03T17:18:26,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/86b9baa9465d403fbfbc892c018d78dc as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/86b9baa9465d403fbfbc892c018d78dc 2024-12-03T17:18:26,016 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/86b9baa9465d403fbfbc892c018d78dc, entries=150, sequenceid=101, filesize=11.7 K 2024-12-03T17:18:26,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/bacea7f9d98f4e399a7af4c655dfce0d as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/bacea7f9d98f4e399a7af4c655dfce0d 2024-12-03T17:18:26,027 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/bacea7f9d98f4e399a7af4c655dfce0d, entries=150, sequenceid=101, filesize=11.7 K 2024-12-03T17:18:26,031 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 3e5f45a8f2579c6bf86ccea0cc76d7f9 in 768ms, sequenceid=101, compaction requested=true 2024-12-03T17:18:26,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3e5f45a8f2579c6bf86ccea0cc76d7f9: 2024-12-03T17:18:26,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3e5f45a8f2579c6bf86ccea0cc76d7f9:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T17:18:26,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:18:26,031 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:18:26,031 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:18:26,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3e5f45a8f2579c6bf86ccea0cc76d7f9:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T17:18:26,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:18:26,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3e5f45a8f2579c6bf86ccea0cc76d7f9:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T17:18:26,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:18:26,032 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:18:26,032 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:18:26,032 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): 3e5f45a8f2579c6bf86ccea0cc76d7f9/A is initiating minor compaction (all files) 2024-12-03T17:18:26,032 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): 3e5f45a8f2579c6bf86ccea0cc76d7f9/B is initiating minor compaction (all files) 2024-12-03T17:18:26,032 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3e5f45a8f2579c6bf86ccea0cc76d7f9/A in TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:26,032 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3e5f45a8f2579c6bf86ccea0cc76d7f9/B in TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:26,032 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/8d8873197a5a446ea8d2b378cb432ffd, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/cabdec57535a431284b3e54e5766c2cc, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/86b9baa9465d403fbfbc892c018d78dc] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp, totalSize=35.3 K 2024-12-03T17:18:26,032 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/2b11daa95f2c4d318052b7aeafee9bf0, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/18ef0e4c158545fdb10e7a4eb4091db9, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/ad4b5c02c88e481f9dd0a3ed9b222efd] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp, totalSize=90.8 K 2024-12-03T17:18:26,032 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:26,032 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. files: [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/2b11daa95f2c4d318052b7aeafee9bf0, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/18ef0e4c158545fdb10e7a4eb4091db9, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/ad4b5c02c88e481f9dd0a3ed9b222efd] 2024-12-03T17:18:26,032 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d8873197a5a446ea8d2b378cb432ffd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733246301692 2024-12-03T17:18:26,034 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting cabdec57535a431284b3e54e5766c2cc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733246302400 2024-12-03T17:18:26,034 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b11daa95f2c4d318052b7aeafee9bf0, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733246301692 2024-12-03T17:18:26,034 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18ef0e4c158545fdb10e7a4eb4091db9, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733246302400 2024-12-03T17:18:26,034 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 86b9baa9465d403fbfbc892c018d78dc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733246304611 2024-12-03T17:18:26,034 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad4b5c02c88e481f9dd0a3ed9b222efd, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733246304611 2024-12-03T17:18:26,048 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3e5f45a8f2579c6bf86ccea0cc76d7f9#B#compaction#488 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:18:26,048 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/19efac42b5904a6ea091de0770f027d1 is 50, key is test_row_0/B:col10/1733246304613/Put/seqid=0 2024-12-03T17:18:26,052 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3e5f45a8f2579c6bf86ccea0cc76d7f9] 2024-12-03T17:18:26,068 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412036c95b0020b30470c96f248f013623e1e_3e5f45a8f2579c6bf86ccea0cc76d7f9 store=[table=TestAcidGuarantees family=A region=3e5f45a8f2579c6bf86ccea0cc76d7f9] 2024-12-03T17:18:26,070 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412036c95b0020b30470c96f248f013623e1e_3e5f45a8f2579c6bf86ccea0cc76d7f9, store=[table=TestAcidGuarantees family=A region=3e5f45a8f2579c6bf86ccea0cc76d7f9] 2024-12-03T17:18:26,070 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412036c95b0020b30470c96f248f013623e1e_3e5f45a8f2579c6bf86ccea0cc76d7f9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3e5f45a8f2579c6bf86ccea0cc76d7f9] 2024-12-03T17:18:26,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742397_1573 (size=12207) 2024-12-03T17:18:26,093 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/19efac42b5904a6ea091de0770f027d1 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/19efac42b5904a6ea091de0770f027d1 2024-12-03T17:18:26,107 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3e5f45a8f2579c6bf86ccea0cc76d7f9/B of 3e5f45a8f2579c6bf86ccea0cc76d7f9 into 19efac42b5904a6ea091de0770f027d1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:18:26,107 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3e5f45a8f2579c6bf86ccea0cc76d7f9: 2024-12-03T17:18:26,107 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., storeName=3e5f45a8f2579c6bf86ccea0cc76d7f9/B, priority=13, startTime=1733246306031; duration=0sec 2024-12-03T17:18:26,107 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:18:26,107 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3e5f45a8f2579c6bf86ccea0cc76d7f9:B 2024-12-03T17:18:26,107 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-03T17:18:26,111 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-03T17:18:26,112 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): 3e5f45a8f2579c6bf86ccea0cc76d7f9/C is initiating minor compaction (all files) 2024-12-03T17:18:26,112 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3e5f45a8f2579c6bf86ccea0cc76d7f9/C in TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:26,112 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/477419cffdba468c8fda655cb669b4ae, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/fe22d51cc264468c91d2e93055a05acf, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/bacea7f9d98f4e399a7af4c655dfce0d] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp, totalSize=35.3 K 2024-12-03T17:18:26,112 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 477419cffdba468c8fda655cb669b4ae, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733246301692 2024-12-03T17:18:26,112 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting fe22d51cc264468c91d2e93055a05acf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733246302400 2024-12-03T17:18:26,113 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting bacea7f9d98f4e399a7af4c655dfce0d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733246304611 2024-12-03T17:18:26,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742398_1574 (size=4469) 2024-12-03T17:18:26,126 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3e5f45a8f2579c6bf86ccea0cc76d7f9#A#compaction#489 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:18:26,126 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/901459cabfdb46588ca60b40109172cf is 175, key is test_row_0/A:col10/1733246304613/Put/seqid=0 2024-12-03T17:18:26,169 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3e5f45a8f2579c6bf86ccea0cc76d7f9#C#compaction#490 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:18:26,169 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/09c26bcff324486c9290ce4a3ee8a9a8 is 50, key is test_row_0/C:col10/1733246304613/Put/seqid=0 2024-12-03T17:18:26,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742399_1575 (size=31161) 2024-12-03T17:18:26,194 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/901459cabfdb46588ca60b40109172cf as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/901459cabfdb46588ca60b40109172cf 2024-12-03T17:18:26,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742400_1576 (size=12207) 2024-12-03T17:18:26,230 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3e5f45a8f2579c6bf86ccea0cc76d7f9/A of 3e5f45a8f2579c6bf86ccea0cc76d7f9 into 901459cabfdb46588ca60b40109172cf(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:18:26,230 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3e5f45a8f2579c6bf86ccea0cc76d7f9: 2024-12-03T17:18:26,230 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., storeName=3e5f45a8f2579c6bf86ccea0cc76d7f9/A, priority=13, startTime=1733246306031; duration=0sec 2024-12-03T17:18:26,230 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:18:26,230 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3e5f45a8f2579c6bf86ccea0cc76d7f9:A 2024-12-03T17:18:26,236 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/09c26bcff324486c9290ce4a3ee8a9a8 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/09c26bcff324486c9290ce4a3ee8a9a8 2024-12-03T17:18:26,273 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3e5f45a8f2579c6bf86ccea0cc76d7f9/C of 3e5f45a8f2579c6bf86ccea0cc76d7f9 into 09c26bcff324486c9290ce4a3ee8a9a8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:18:26,273 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3e5f45a8f2579c6bf86ccea0cc76d7f9: 2024-12-03T17:18:26,273 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., storeName=3e5f45a8f2579c6bf86ccea0cc76d7f9/C, priority=13, startTime=1733246306031; duration=0sec 2024-12-03T17:18:26,273 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:18:26,273 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3e5f45a8f2579c6bf86ccea0cc76d7f9:C 2024-12-03T17:18:26,504 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3e5f45a8f2579c6bf86ccea0cc76d7f9 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-03T17:18:26,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:26,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e5f45a8f2579c6bf86ccea0cc76d7f9, store=A 2024-12-03T17:18:26,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:26,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e5f45a8f2579c6bf86ccea0cc76d7f9, store=B 2024-12-03T17:18:26,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:26,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e5f45a8f2579c6bf86ccea0cc76d7f9, store=C 2024-12-03T17:18:26,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:26,525 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203044153809afe433f96f777b8bf68c885_3e5f45a8f2579c6bf86ccea0cc76d7f9 is 50, key is test_row_0/A:col10/1733246305321/Put/seqid=0 2024-12-03T17:18:26,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:26,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246366529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:26,538 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:26,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:26,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:26,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:26,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34552 deadline: 1733246366531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:26,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:26,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:26,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:26,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:26,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246366539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:26,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:26,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246366534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:26,546 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:26,546 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:26,546 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:26,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:26,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:26,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:26,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742401_1577 (size=12154) 2024-12-03T17:18:26,571 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:18:26,596 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203044153809afe433f96f777b8bf68c885_3e5f45a8f2579c6bf86ccea0cc76d7f9 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203044153809afe433f96f777b8bf68c885_3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:26,597 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/a35336f06b19419ca2d77ea98304ec26, store: [table=TestAcidGuarantees family=A region=3e5f45a8f2579c6bf86ccea0cc76d7f9] 2024-12-03T17:18:26,597 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/a35336f06b19419ca2d77ea98304ec26 is 175, key is test_row_0/A:col10/1733246305321/Put/seqid=0 2024-12-03T17:18:26,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742402_1578 (size=30955) 2024-12-03T17:18:26,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:26,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246366644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:26,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:26,647 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:26,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34552 deadline: 1733246366644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:26,647 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:26,647 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:26,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:26,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:26,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:26,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:26,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246366656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:26,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:26,657 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:26,657 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:26,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246366656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:26,657 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:26,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:26,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:26,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:26,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-03T17:18:26,695 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:18:26,701 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T17:18:26,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-12-03T17:18:26,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-03T17:18:26,722 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T17:18:26,725 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T17:18:26,725 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T17:18:26,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-03T17:18:26,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:26,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246366855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:26,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:26,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34552 deadline: 1733246366856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:26,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:26,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:26,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:26,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:26,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:26,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:26,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:26,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246366864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:26,866 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:26,866 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:26,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:26,866 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:26,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246366865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:26,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:26,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:26,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:26,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-03T17:18:26,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:26,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. as already flushing 2024-12-03T17:18:26,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:26,883 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:26,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:26,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:27,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-03T17:18:27,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-03T17:18:27,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:27,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. as already flushing 2024-12-03T17:18:27,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:27,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:27,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:27,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:27,051 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=123, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/a35336f06b19419ca2d77ea98304ec26 2024-12-03T17:18:27,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/69738a468bd341c2946ebbdda1064307 is 50, key is test_row_0/B:col10/1733246305321/Put/seqid=0 2024-12-03T17:18:27,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742403_1579 (size=12001) 2024-12-03T17:18:27,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:27,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246367164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:27,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:27,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34552 deadline: 1733246367164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:27,169 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:27,169 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:27,169 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:27,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:27,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:27,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:27,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:27,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246367174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:27,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:27,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:27,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:27,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:27,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246367179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:27,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:27,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:27,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:27,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-03T17:18:27,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:27,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. as already flushing 2024-12-03T17:18:27,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:27,202 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:27,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:27,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:27,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-03T17:18:27,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-03T17:18:27,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:27,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. as already flushing 2024-12-03T17:18:27,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:27,363 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:27,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:27,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:27,495 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/69738a468bd341c2946ebbdda1064307 2024-12-03T17:18:27,504 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/3401c18fa29c42379c34ce70e2684e5f is 50, key is test_row_0/C:col10/1733246305321/Put/seqid=0 2024-12-03T17:18:27,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-03T17:18:27,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:27,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. as already flushing 2024-12-03T17:18:27,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:27,524 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:27,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:27,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:27,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742404_1580 (size=12001) 2024-12-03T17:18:27,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:27,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34552 deadline: 1733246367674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:27,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:27,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246367675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:27,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-03T17:18:27,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:27,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:27,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:27,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:27,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. as already flushing 2024-12-03T17:18:27,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:27,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:27,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:27,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:27,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:27,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:27,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:27,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:27,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246367683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:27,688 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:27,688 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:27,688 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:27,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:27,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246367694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:27,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:27,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:27,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:27,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-03T17:18:27,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:27,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. as already flushing 2024-12-03T17:18:27,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:27,838 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:27,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:27,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:27,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-03T17:18:27,945 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/3401c18fa29c42379c34ce70e2684e5f 2024-12-03T17:18:27,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/a35336f06b19419ca2d77ea98304ec26 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/a35336f06b19419ca2d77ea98304ec26 2024-12-03T17:18:27,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-03T17:18:27,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:27,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. as already flushing 2024-12-03T17:18:27,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:27,996 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:27,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:27,997 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/a35336f06b19419ca2d77ea98304ec26, entries=150, sequenceid=123, filesize=30.2 K 2024-12-03T17:18:28,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/69738a468bd341c2946ebbdda1064307 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/69738a468bd341c2946ebbdda1064307 2024-12-03T17:18:28,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:28,028 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/69738a468bd341c2946ebbdda1064307, entries=150, sequenceid=123, filesize=11.7 K 2024-12-03T17:18:28,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/3401c18fa29c42379c34ce70e2684e5f as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/3401c18fa29c42379c34ce70e2684e5f 2024-12-03T17:18:28,046 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/3401c18fa29c42379c34ce70e2684e5f, entries=150, sequenceid=123, filesize=11.7 K 2024-12-03T17:18:28,047 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 3e5f45a8f2579c6bf86ccea0cc76d7f9 in 1543ms, sequenceid=123, compaction requested=false 2024-12-03T17:18:28,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3e5f45a8f2579c6bf86ccea0cc76d7f9: 2024-12-03T17:18:28,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-03T17:18:28,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:28,157 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing 3e5f45a8f2579c6bf86ccea0cc76d7f9 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-03T17:18:28,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e5f45a8f2579c6bf86ccea0cc76d7f9, store=A 2024-12-03T17:18:28,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:28,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e5f45a8f2579c6bf86ccea0cc76d7f9, store=B 2024-12-03T17:18:28,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:28,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e5f45a8f2579c6bf86ccea0cc76d7f9, store=C 2024-12-03T17:18:28,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:28,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203fdb5c1255e2141a6a703cb10cd09a6d6_3e5f45a8f2579c6bf86ccea0cc76d7f9 is 50, key is test_row_0/A:col10/1733246306533/Put/seqid=0 2024-12-03T17:18:28,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742405_1581 (size=12304) 2024-12-03T17:18:28,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:18:28,612 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203fdb5c1255e2141a6a703cb10cd09a6d6_3e5f45a8f2579c6bf86ccea0cc76d7f9 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203fdb5c1255e2141a6a703cb10cd09a6d6_3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:28,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/84df52f6296e4cd690ac87dfed412292, store: [table=TestAcidGuarantees family=A region=3e5f45a8f2579c6bf86ccea0cc76d7f9] 2024-12-03T17:18:28,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/84df52f6296e4cd690ac87dfed412292 is 175, key is test_row_0/A:col10/1733246306533/Put/seqid=0 2024-12-03T17:18:28,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742406_1582 (size=31105) 2024-12-03T17:18:28,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. as already flushing 2024-12-03T17:18:28,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:28,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:28,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246368788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:28,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:28,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34634 deadline: 1733246368791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:28,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:28,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246368795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:28,802 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:28,802 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:28,802 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:28,802 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:28,802 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:28,802 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:28,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:28,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246368796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:28,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:28,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:28,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:28,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:28,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:28,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:28,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:28,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34552 deadline: 1733246368802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:28,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:28,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:28,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:28,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-03T17:18:28,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:28,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246368914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:28,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:28,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:28,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:28,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:28,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246368914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:28,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:28,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246368914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:28,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:28,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:28,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34634 deadline: 1733246368916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:28,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:28,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:28,927 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:28,927 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:28,927 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:28,930 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:28,930 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:28,930 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:28,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:28,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34552 deadline: 1733246368925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:28,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:28,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:28,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:29,042 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=136, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/84df52f6296e4cd690ac87dfed412292 2024-12-03T17:18:29,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/07f451c0a55c4f398131bea18b17c171 is 50, key is test_row_0/B:col10/1733246306533/Put/seqid=0 2024-12-03T17:18:29,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742407_1583 (size=12151) 2024-12-03T17:18:29,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:29,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246369124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:29,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:29,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:29,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246369124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:29,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:29,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:29,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:29,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:29,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:29,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:29,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246369133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:29,135 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:29,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:29,135 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:29,135 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:29,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34634 deadline: 1733246369133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:29,136 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:29,136 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:29,136 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:29,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:29,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34552 deadline: 1733246369144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:29,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:29,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:29,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:29,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:29,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246369434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:29,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:29,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246369434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:29,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:29,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:29,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:29,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:29,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:29,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:29,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:29,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246369443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:29,448 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:29,448 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:29,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:29,448 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:29,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34634 deadline: 1733246369443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:29,448 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:29,448 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:29,448 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:29,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:29,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34552 deadline: 1733246369465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:29,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:29,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:29,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:29,494 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/07f451c0a55c4f398131bea18b17c171 2024-12-03T17:18:29,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/a95eb3f025b14d168ad619ebe0afb0aa is 50, key is test_row_0/C:col10/1733246306533/Put/seqid=0 2024-12-03T17:18:29,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742408_1584 (size=12151) 2024-12-03T17:18:29,573 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/a95eb3f025b14d168ad619ebe0afb0aa 2024-12-03T17:18:29,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/84df52f6296e4cd690ac87dfed412292 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/84df52f6296e4cd690ac87dfed412292 2024-12-03T17:18:29,597 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/84df52f6296e4cd690ac87dfed412292, entries=150, sequenceid=136, filesize=30.4 K 2024-12-03T17:18:29,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/07f451c0a55c4f398131bea18b17c171 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/07f451c0a55c4f398131bea18b17c171 2024-12-03T17:18:29,607 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/07f451c0a55c4f398131bea18b17c171, entries=150, sequenceid=136, filesize=11.9 K 2024-12-03T17:18:29,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/a95eb3f025b14d168ad619ebe0afb0aa as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/a95eb3f025b14d168ad619ebe0afb0aa 2024-12-03T17:18:29,613 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/a95eb3f025b14d168ad619ebe0afb0aa, entries=150, sequenceid=136, filesize=11.9 K 2024-12-03T17:18:29,617 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 3e5f45a8f2579c6bf86ccea0cc76d7f9 in 1461ms, sequenceid=136, compaction requested=true 2024-12-03T17:18:29,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for 3e5f45a8f2579c6bf86ccea0cc76d7f9: 2024-12-03T17:18:29,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:29,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-12-03T17:18:29,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-12-03T17:18:29,638 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-12-03T17:18:29,639 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8980 sec 2024-12-03T17:18:29,652 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 2.9450 sec 2024-12-03T17:18:29,949 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3e5f45a8f2579c6bf86ccea0cc76d7f9 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T17:18:29,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:29,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e5f45a8f2579c6bf86ccea0cc76d7f9, store=A 2024-12-03T17:18:29,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:18:29,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e5f45a8f2579c6bf86ccea0cc76d7f9, store=B 2024-12-03T17:18:29,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:18:29,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e5f45a8f2579c6bf86ccea0cc76d7f9, store=C 2024-12-03T17:18:29,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-03T17:18:29,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ecf4fa6dfe0442b28fd387e1a4ba389e_3e5f45a8f2579c6bf86ccea0cc76d7f9 is 50, key is test_row_0/A:col10/1733246309947/Put/seqid=0 2024-12-03T17:18:29,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:29,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246369967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:29,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:29,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:29,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:29,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:29,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:29,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34634 deadline: 1733246369977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:29,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246369974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:29,988 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:29,988 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:29,988 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:29,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:29,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:29,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:29,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:29,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246369983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:29,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:29,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34552 deadline: 1733246369986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:29,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:29,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:29,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:29,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:29,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:29,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:30,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742409_1585 (size=14794) 2024-12-03T17:18:30,011 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:18:30,018 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203ecf4fa6dfe0442b28fd387e1a4ba389e_3e5f45a8f2579c6bf86ccea0cc76d7f9 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ecf4fa6dfe0442b28fd387e1a4ba389e_3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:30,021 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/4bd4ef0534d749f3bd763c66236a73a8, store: [table=TestAcidGuarantees family=A region=3e5f45a8f2579c6bf86ccea0cc76d7f9] 2024-12-03T17:18:30,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/4bd4ef0534d749f3bd763c66236a73a8 is 175, key is test_row_0/A:col10/1733246309947/Put/seqid=0 2024-12-03T17:18:30,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742410_1586 (size=39749) 2024-12-03T17:18:30,035 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/4bd4ef0534d749f3bd763c66236a73a8 2024-12-03T17:18:30,065 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/0e005ddf818f4da3813796b18828156a is 50, key is test_row_0/B:col10/1733246309947/Put/seqid=0 2024-12-03T17:18:30,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:30,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246370087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:30,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742411_1587 (size=12151) 2024-12-03T17:18:30,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:30,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:30,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:30,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:30,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246370093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:30,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:30,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:30,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:30,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:30,100 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/0e005ddf818f4da3813796b18828156a 2024-12-03T17:18:30,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246370094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:30,100 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:30,100 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:30,100 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:30,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:30,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34634 deadline: 1733246370094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:30,102 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:30,102 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:30,102 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:30,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/9c48f254cbad4ffaa1ca82571032c56b is 50, key is test_row_0/C:col10/1733246309947/Put/seqid=0 2024-12-03T17:18:30,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742412_1588 (size=12151) 2024-12-03T17:18:30,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:30,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246370303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:30,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:30,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246370303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:30,308 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:30,308 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:30,308 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:30,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:30,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246370303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:30,308 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:30,308 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:30,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:30,308 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:30,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:30,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:30,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:30,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34634 deadline: 1733246370315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:30,316 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:30,316 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:30,316 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:30,612 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/9c48f254cbad4ffaa1ca82571032c56b 2024-12-03T17:18:30,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:30,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246370614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:30,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:30,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246370614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:30,621 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:30,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:30,621 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:30,621 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:30,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246370614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:30,621 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:30,621 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:30,621 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:30,622 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:30,622 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:30,622 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:30,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/4bd4ef0534d749f3bd763c66236a73a8 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/4bd4ef0534d749f3bd763c66236a73a8 2024-12-03T17:18:30,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:30,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34634 deadline: 1733246370625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:30,633 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:30,633 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:30,633 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:30,635 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/4bd4ef0534d749f3bd763c66236a73a8, entries=200, sequenceid=157, filesize=38.8 K 2024-12-03T17:18:30,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/0e005ddf818f4da3813796b18828156a as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/0e005ddf818f4da3813796b18828156a 2024-12-03T17:18:30,644 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/0e005ddf818f4da3813796b18828156a, entries=150, sequenceid=157, filesize=11.9 K 2024-12-03T17:18:30,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/9c48f254cbad4ffaa1ca82571032c56b as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/9c48f254cbad4ffaa1ca82571032c56b 2024-12-03T17:18:30,659 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/9c48f254cbad4ffaa1ca82571032c56b, entries=150, sequenceid=157, filesize=11.9 K 2024-12-03T17:18:30,660 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 3e5f45a8f2579c6bf86ccea0cc76d7f9 in 711ms, sequenceid=157, compaction requested=true 2024-12-03T17:18:30,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3e5f45a8f2579c6bf86ccea0cc76d7f9: 2024-12-03T17:18:30,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3e5f45a8f2579c6bf86ccea0cc76d7f9:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T17:18:30,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:18:30,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3e5f45a8f2579c6bf86ccea0cc76d7f9:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T17:18:30,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:18:30,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3e5f45a8f2579c6bf86ccea0cc76d7f9:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T17:18:30,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-03T17:18:30,661 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T17:18:30,661 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T17:18:30,668 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132970 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T17:18:30,668 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): 3e5f45a8f2579c6bf86ccea0cc76d7f9/A is initiating minor compaction (all files) 2024-12-03T17:18:30,668 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3e5f45a8f2579c6bf86ccea0cc76d7f9/A in TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:30,669 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/901459cabfdb46588ca60b40109172cf, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/a35336f06b19419ca2d77ea98304ec26, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/84df52f6296e4cd690ac87dfed412292, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/4bd4ef0534d749f3bd763c66236a73a8] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp, totalSize=129.9 K 2024-12-03T17:18:30,669 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:30,669 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. files: [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/901459cabfdb46588ca60b40109172cf, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/a35336f06b19419ca2d77ea98304ec26, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/84df52f6296e4cd690ac87dfed412292, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/4bd4ef0534d749f3bd763c66236a73a8] 2024-12-03T17:18:30,669 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T17:18:30,669 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): 3e5f45a8f2579c6bf86ccea0cc76d7f9/B is initiating minor compaction (all files) 2024-12-03T17:18:30,669 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3e5f45a8f2579c6bf86ccea0cc76d7f9/B in TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:30,669 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/19efac42b5904a6ea091de0770f027d1, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/69738a468bd341c2946ebbdda1064307, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/07f451c0a55c4f398131bea18b17c171, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/0e005ddf818f4da3813796b18828156a] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp, totalSize=47.4 K 2024-12-03T17:18:30,669 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 901459cabfdb46588ca60b40109172cf, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733246304611 2024-12-03T17:18:30,670 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 19efac42b5904a6ea091de0770f027d1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733246304611 2024-12-03T17:18:30,670 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting a35336f06b19419ca2d77ea98304ec26, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1733246305321 2024-12-03T17:18:30,670 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 69738a468bd341c2946ebbdda1064307, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1733246305321 2024-12-03T17:18:30,670 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84df52f6296e4cd690ac87dfed412292, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733246306521 2024-12-03T17:18:30,670 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 07f451c0a55c4f398131bea18b17c171, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733246306521 2024-12-03T17:18:30,670 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e005ddf818f4da3813796b18828156a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733246308783 2024-12-03T17:18:30,670 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4bd4ef0534d749f3bd763c66236a73a8, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733246308783 2024-12-03T17:18:30,683 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3e5f45a8f2579c6bf86ccea0cc76d7f9] 2024-12-03T17:18:30,690 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3e5f45a8f2579c6bf86ccea0cc76d7f9#B#compaction#501 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:18:30,690 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/7aa06e2d24ca4640b966889a17eb4dd9 is 50, key is test_row_0/B:col10/1733246309947/Put/seqid=0 2024-12-03T17:18:30,701 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412038f7bdb4b52b142218fc2d9040b634fcc_3e5f45a8f2579c6bf86ccea0cc76d7f9 store=[table=TestAcidGuarantees family=A region=3e5f45a8f2579c6bf86ccea0cc76d7f9] 2024-12-03T17:18:30,704 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412038f7bdb4b52b142218fc2d9040b634fcc_3e5f45a8f2579c6bf86ccea0cc76d7f9, store=[table=TestAcidGuarantees family=A region=3e5f45a8f2579c6bf86ccea0cc76d7f9] 2024-12-03T17:18:30,704 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412038f7bdb4b52b142218fc2d9040b634fcc_3e5f45a8f2579c6bf86ccea0cc76d7f9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3e5f45a8f2579c6bf86ccea0cc76d7f9] 2024-12-03T17:18:30,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742413_1589 (size=12493) 2024-12-03T17:18:30,753 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/7aa06e2d24ca4640b966889a17eb4dd9 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/7aa06e2d24ca4640b966889a17eb4dd9 2024-12-03T17:18:30,758 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3e5f45a8f2579c6bf86ccea0cc76d7f9/B of 3e5f45a8f2579c6bf86ccea0cc76d7f9 into 7aa06e2d24ca4640b966889a17eb4dd9(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:18:30,759 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3e5f45a8f2579c6bf86ccea0cc76d7f9: 2024-12-03T17:18:30,759 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., storeName=3e5f45a8f2579c6bf86ccea0cc76d7f9/B, priority=12, startTime=1733246310661; duration=0sec 2024-12-03T17:18:30,759 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:18:30,759 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3e5f45a8f2579c6bf86ccea0cc76d7f9:B 2024-12-03T17:18:30,759 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T17:18:30,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742414_1590 (size=4469) 2024-12-03T17:18:30,762 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T17:18:30,762 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): 3e5f45a8f2579c6bf86ccea0cc76d7f9/C is initiating minor compaction (all files) 2024-12-03T17:18:30,762 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3e5f45a8f2579c6bf86ccea0cc76d7f9/C in TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:30,762 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/09c26bcff324486c9290ce4a3ee8a9a8, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/3401c18fa29c42379c34ce70e2684e5f, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/a95eb3f025b14d168ad619ebe0afb0aa, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/9c48f254cbad4ffaa1ca82571032c56b] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp, totalSize=47.4 K 2024-12-03T17:18:30,763 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 09c26bcff324486c9290ce4a3ee8a9a8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1733246304611 2024-12-03T17:18:30,763 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3e5f45a8f2579c6bf86ccea0cc76d7f9#A#compaction#500 average throughput is 0.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:18:30,763 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 3401c18fa29c42379c34ce70e2684e5f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1733246305321 2024-12-03T17:18:30,764 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/158f9f06ad7f4c3ba01a0f99522d4dc3 is 175, key is test_row_0/A:col10/1733246309947/Put/seqid=0 2024-12-03T17:18:30,764 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting a95eb3f025b14d168ad619ebe0afb0aa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733246306521 2024-12-03T17:18:30,764 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c48f254cbad4ffaa1ca82571032c56b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733246308783 2024-12-03T17:18:30,798 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3e5f45a8f2579c6bf86ccea0cc76d7f9#C#compaction#502 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:18:30,798 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/5f68fdf67b8e4f9db4d917b44026058e is 50, key is test_row_0/C:col10/1733246309947/Put/seqid=0 2024-12-03T17:18:30,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742415_1591 (size=31447) 2024-12-03T17:18:30,816 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/158f9f06ad7f4c3ba01a0f99522d4dc3 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/158f9f06ad7f4c3ba01a0f99522d4dc3 2024-12-03T17:18:30,823 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3e5f45a8f2579c6bf86ccea0cc76d7f9/A of 3e5f45a8f2579c6bf86ccea0cc76d7f9 into 158f9f06ad7f4c3ba01a0f99522d4dc3(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:18:30,823 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3e5f45a8f2579c6bf86ccea0cc76d7f9: 2024-12-03T17:18:30,823 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., storeName=3e5f45a8f2579c6bf86ccea0cc76d7f9/A, priority=12, startTime=1733246310661; duration=0sec 2024-12-03T17:18:30,823 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:18:30,823 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3e5f45a8f2579c6bf86ccea0cc76d7f9:A 2024-12-03T17:18:30,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742416_1592 (size=12493) 2024-12-03T17:18:30,871 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/5f68fdf67b8e4f9db4d917b44026058e as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/5f68fdf67b8e4f9db4d917b44026058e 2024-12-03T17:18:30,883 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3e5f45a8f2579c6bf86ccea0cc76d7f9/C of 3e5f45a8f2579c6bf86ccea0cc76d7f9 into 5f68fdf67b8e4f9db4d917b44026058e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:18:30,883 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3e5f45a8f2579c6bf86ccea0cc76d7f9: 2024-12-03T17:18:30,883 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., storeName=3e5f45a8f2579c6bf86ccea0cc76d7f9/C, priority=12, startTime=1733246310661; duration=0sec 2024-12-03T17:18:30,883 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:18:30,883 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3e5f45a8f2579c6bf86ccea0cc76d7f9:C 2024-12-03T17:18:30,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-03T17:18:30,884 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:18:30,885 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T17:18:30,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-12-03T17:18:30,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T17:18:30,893 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T17:18:30,895 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T17:18:30,895 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T17:18:31,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T17:18:31,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:31,008 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3e5f45a8f2579c6bf86ccea0cc76d7f9 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-03T17:18:31,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e5f45a8f2579c6bf86ccea0cc76d7f9, store=A 2024-12-03T17:18:31,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:31,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e5f45a8f2579c6bf86ccea0cc76d7f9, store=B 2024-12-03T17:18:31,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:31,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e5f45a8f2579c6bf86ccea0cc76d7f9, store=C 2024-12-03T17:18:31,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:31,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203286097c1986c46f2b57e6727924c7a5a_3e5f45a8f2579c6bf86ccea0cc76d7f9 is 50, key is test_row_0/A:col10/1733246309969/Put/seqid=0 2024-12-03T17:18:31,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-03T17:18:31,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:31,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. as already flushing 2024-12-03T17:18:31,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:31,051 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:31,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:31,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:31,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742417_1593 (size=14794) 2024-12-03T17:18:31,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:31,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34634 deadline: 1733246371143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:31,159 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:31,159 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:31,159 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:31,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:31,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34552 deadline: 1733246371143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:31,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:31,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:31,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:31,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:31,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246371151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:31,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:31,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:31,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:31,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:31,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246371152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:31,164 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:31,165 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:31,165 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:31,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:31,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246371155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:31,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:31,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:31,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:31,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-03T17:18:31,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T17:18:31,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:31,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. as already flushing 2024-12-03T17:18:31,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:31,214 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:31,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:31,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:31,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:31,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34552 deadline: 1733246371264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:31,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:31,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:31,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:31,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:31,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246371264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:31,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:31,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:31,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:31,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:31,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246371274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:31,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:31,279 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:31,279 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:31,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246371274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:31,279 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:31,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:31,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:31,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:31,375 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-03T17:18:31,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:31,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. as already flushing 2024-12-03T17:18:31,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:31,375 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:31,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:31,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:31,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:31,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246371474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:31,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:31,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:31,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:31,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:31,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34552 deadline: 1733246371474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:31,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:31,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:31,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:31,482 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:18:31,487 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203286097c1986c46f2b57e6727924c7a5a_3e5f45a8f2579c6bf86ccea0cc76d7f9 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203286097c1986c46f2b57e6727924c7a5a_3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:31,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:31,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246371486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:31,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:31,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246371486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:31,492 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:31,492 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:31,492 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:31,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:31,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:31,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:31,498 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/d0fc1619dcc74efb88cb697af66bc562, store: [table=TestAcidGuarantees family=A region=3e5f45a8f2579c6bf86ccea0cc76d7f9] 2024-12-03T17:18:31,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/d0fc1619dcc74efb88cb697af66bc562 is 175, key is test_row_0/A:col10/1733246309969/Put/seqid=0 2024-12-03T17:18:31,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T17:18:31,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-03T17:18:31,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:31,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. as already flushing 2024-12-03T17:18:31,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:31,538 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:31,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:31,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:31,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742418_1594 (size=39749) 2024-12-03T17:18:31,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-03T17:18:31,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:31,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. as already flushing 2024-12-03T17:18:31,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:31,695 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:31,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:31,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:31,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:31,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246371784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:31,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:31,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34552 deadline: 1733246371784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:31,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:31,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:31,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:31,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:31,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:31,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:31,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:31,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246371803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:31,809 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:31,809 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:31,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:31,809 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:31,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246371804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:31,810 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:31,810 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:31,810 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:31,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-03T17:18:31,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:31,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. as already flushing 2024-12-03T17:18:31,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:31,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:31,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:31,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:31,963 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=172, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/d0fc1619dcc74efb88cb697af66bc562 2024-12-03T17:18:31,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/29751bc810414d5d9c69e648609a86eb is 50, key is test_row_0/B:col10/1733246309969/Put/seqid=0 2024-12-03T17:18:32,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-03T17:18:32,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:32,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. as already flushing 2024-12-03T17:18:32,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:32,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:32,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:32,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:32,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T17:18:32,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742419_1595 (size=12151) 2024-12-03T17:18:32,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-03T17:18:32,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:32,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. as already flushing 2024-12-03T17:18:32,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:32,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:32,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:32,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:32,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:32,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34634 deadline: 1733246372180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:32,181 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 27e0f852a164,41167,1733246151593 for 'test_row_2' in 3e5f45a8f2579c6bf86ccea0cc76d7f9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T17:18:32,182 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:32,182 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:32,182 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:32,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:32,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34552 deadline: 1733246372293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:32,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:32,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:32,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:32,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:32,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246372293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:32,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:32,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:32,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:32,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:32,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246372314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:32,321 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:32,321 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:32,321 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:32,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-03T17:18:32,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:32,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. as already flushing 2024-12-03T17:18:32,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:32,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:32,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:32,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:32,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246372323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:32,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:32,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:32,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:32,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:32,438 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/29751bc810414d5d9c69e648609a86eb 2024-12-03T17:18:32,494 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-03T17:18:32,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:32,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. as already flushing 2024-12-03T17:18:32,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:32,496 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:32,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:32,496 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/803719d3635947ccb29019f984071d54 is 50, key is test_row_0/C:col10/1733246309969/Put/seqid=0 2024-12-03T17:18:32,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:32,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742420_1596 (size=12151) 2024-12-03T17:18:32,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-03T17:18:32,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:32,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. as already flushing 2024-12-03T17:18:32,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:32,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:32,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:32,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:32,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-03T17:18:32,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:32,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. as already flushing 2024-12-03T17:18:32,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:32,818 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:32,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:32,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:32,915 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/803719d3635947ccb29019f984071d54 2024-12-03T17:18:32,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/d0fc1619dcc74efb88cb697af66bc562 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/d0fc1619dcc74efb88cb697af66bc562 2024-12-03T17:18:32,963 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/d0fc1619dcc74efb88cb697af66bc562, entries=200, sequenceid=172, filesize=38.8 K 2024-12-03T17:18:32,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/29751bc810414d5d9c69e648609a86eb as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/29751bc810414d5d9c69e648609a86eb 2024-12-03T17:18:32,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-03T17:18:32,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:32,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. as already flushing 2024-12-03T17:18:32,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:32,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:32,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:32,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:32,992 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/29751bc810414d5d9c69e648609a86eb, entries=150, sequenceid=172, filesize=11.9 K 2024-12-03T17:18:32,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/803719d3635947ccb29019f984071d54 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/803719d3635947ccb29019f984071d54 2024-12-03T17:18:33,031 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/803719d3635947ccb29019f984071d54, entries=150, sequenceid=172, filesize=11.9 K 2024-12-03T17:18:33,042 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 3e5f45a8f2579c6bf86ccea0cc76d7f9 in 2034ms, sequenceid=172, compaction requested=false 2024-12-03T17:18:33,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3e5f45a8f2579c6bf86ccea0cc76d7f9: 2024-12-03T17:18:33,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T17:18:33,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-03T17:18:33,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:33,132 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing 3e5f45a8f2579c6bf86ccea0cc76d7f9 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-03T17:18:33,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e5f45a8f2579c6bf86ccea0cc76d7f9, store=A 2024-12-03T17:18:33,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:33,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e5f45a8f2579c6bf86ccea0cc76d7f9, store=B 2024-12-03T17:18:33,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:33,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e5f45a8f2579c6bf86ccea0cc76d7f9, store=C 2024-12-03T17:18:33,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:33,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412030270750143124e8d8d5ae07dff6bc824_3e5f45a8f2579c6bf86ccea0cc76d7f9 is 50, key is test_row_0/A:col10/1733246311149/Put/seqid=0 2024-12-03T17:18:33,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742421_1597 (size=12304) 2024-12-03T17:18:33,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:18:33,257 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412030270750143124e8d8d5ae07dff6bc824_3e5f45a8f2579c6bf86ccea0cc76d7f9 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412030270750143124e8d8d5ae07dff6bc824_3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:33,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/d099eda1d9d44fc68cce3682cc9c697a, store: [table=TestAcidGuarantees family=A region=3e5f45a8f2579c6bf86ccea0cc76d7f9] 2024-12-03T17:18:33,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/d099eda1d9d44fc68cce3682cc9c697a is 175, key is test_row_0/A:col10/1733246311149/Put/seqid=0 2024-12-03T17:18:33,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742422_1598 (size=31105) 2024-12-03T17:18:33,304 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=192, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/d099eda1d9d44fc68cce3682cc9c697a 2024-12-03T17:18:33,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/8b31079b0a5c4202a43a68316da8d380 is 50, key is test_row_0/B:col10/1733246311149/Put/seqid=0 2024-12-03T17:18:33,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. as already flushing 2024-12-03T17:18:33,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:33,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742423_1599 (size=12151) 2024-12-03T17:18:33,345 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/8b31079b0a5c4202a43a68316da8d380 2024-12-03T17:18:33,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/d0786d5332d841bc942184191a12e3e6 is 50, key is test_row_0/C:col10/1733246311149/Put/seqid=0 2024-12-03T17:18:33,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742424_1600 (size=12151) 2024-12-03T17:18:33,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:33,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34552 deadline: 1733246373356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:33,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:33,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:33,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:33,364 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/d0786d5332d841bc942184191a12e3e6 2024-12-03T17:18:33,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/d099eda1d9d44fc68cce3682cc9c697a as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/d099eda1d9d44fc68cce3682cc9c697a 2024-12-03T17:18:33,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:33,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246373362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:33,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:33,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246373364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:33,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:33,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246373364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:33,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:33,370 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:33,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:33,370 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:33,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:33,370 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:33,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:33,371 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:33,371 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:33,373 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/d099eda1d9d44fc68cce3682cc9c697a, entries=150, sequenceid=192, filesize=30.4 K 2024-12-03T17:18:33,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/8b31079b0a5c4202a43a68316da8d380 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/8b31079b0a5c4202a43a68316da8d380 2024-12-03T17:18:33,380 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/8b31079b0a5c4202a43a68316da8d380, entries=150, sequenceid=192, filesize=11.9 K 2024-12-03T17:18:33,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/d0786d5332d841bc942184191a12e3e6 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/d0786d5332d841bc942184191a12e3e6 2024-12-03T17:18:33,387 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/d0786d5332d841bc942184191a12e3e6, entries=150, sequenceid=192, filesize=11.9 K 2024-12-03T17:18:33,388 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 3e5f45a8f2579c6bf86ccea0cc76d7f9 in 256ms, sequenceid=192, compaction requested=true 2024-12-03T17:18:33,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for 3e5f45a8f2579c6bf86ccea0cc76d7f9: 2024-12-03T17:18:33,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:33,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-12-03T17:18:33,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-12-03T17:18:33,399 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-12-03T17:18:33,399 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5000 sec 2024-12-03T17:18:33,404 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 2.5170 sec 2024-12-03T17:18:33,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:33,476 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3e5f45a8f2579c6bf86ccea0cc76d7f9 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-03T17:18:33,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e5f45a8f2579c6bf86ccea0cc76d7f9, store=A 2024-12-03T17:18:33,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:33,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e5f45a8f2579c6bf86ccea0cc76d7f9, store=B 2024-12-03T17:18:33,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:33,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e5f45a8f2579c6bf86ccea0cc76d7f9, store=C 2024-12-03T17:18:33,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:33,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412031699372b192b43569d36f71ddde39c5f_3e5f45a8f2579c6bf86ccea0cc76d7f9 is 50, key is test_row_0/A:col10/1733246313339/Put/seqid=0 2024-12-03T17:18:33,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:33,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34552 deadline: 1733246373511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:33,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:33,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:33,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246373511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:33,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:33,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:33,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:33,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:33,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:33,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246373512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:33,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:33,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:33,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:33,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:33,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:33,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246373520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:33,527 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:33,527 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:33,527 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:33,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742425_1601 (size=12304) 2024-12-03T17:18:33,555 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:18:33,560 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412031699372b192b43569d36f71ddde39c5f_3e5f45a8f2579c6bf86ccea0cc76d7f9 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412031699372b192b43569d36f71ddde39c5f_3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:33,561 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/2d948ec6cc1641a7b02fa50d41c77f29, store: [table=TestAcidGuarantees family=A region=3e5f45a8f2579c6bf86ccea0cc76d7f9] 2024-12-03T17:18:33,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/2d948ec6cc1641a7b02fa50d41c77f29 is 175, key is test_row_0/A:col10/1733246313339/Put/seqid=0 2024-12-03T17:18:33,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742426_1602 (size=31105) 2024-12-03T17:18:33,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:33,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34552 deadline: 1733246373624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:33,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:33,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246373624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:33,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:33,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:33,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:33,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:33,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246373624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:33,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:33,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:33,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:33,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:33,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:33,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:33,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:33,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246373633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:33,638 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:33,639 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:33,639 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:33,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:33,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246373834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:33,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:33,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34552 deadline: 1733246373834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:33,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:33,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246373834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:33,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:33,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:33,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:33,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:33,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:33,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:33,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:33,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:33,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:33,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:33,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246373846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:33,854 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:33,854 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:33,854 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:33,991 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=206, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/2d948ec6cc1641a7b02fa50d41c77f29 2024-12-03T17:18:33,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/73a19e7e0edc4b6f9e4d5fe8e875388f is 50, key is test_row_0/B:col10/1733246313339/Put/seqid=0 2024-12-03T17:18:34,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742427_1603 (size=12151) 2024-12-03T17:18:34,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:34,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246374143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:34,146 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:34,146 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:34,147 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:34,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:34,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34552 deadline: 1733246374144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:34,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:34,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:34,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:34,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:34,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246374154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:34,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:34,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:34,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:34,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:34,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246374163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:34,166 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:34,166 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:34,166 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:34,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:34,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34634 deadline: 1733246374194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:34,200 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 27e0f852a164,41167,1733246151593 for 'test_row_2' in 3e5f45a8f2579c6bf86ccea0cc76d7f9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T17:18:34,201 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:34,201 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:34,201 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:34,402 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/73a19e7e0edc4b6f9e4d5fe8e875388f 2024-12-03T17:18:34,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/3e04cae0db914220becbae247176ae67 is 50, key is test_row_0/C:col10/1733246313339/Put/seqid=0 2024-12-03T17:18:34,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742428_1604 (size=12151) 2024-12-03T17:18:34,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:34,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34552 deadline: 1733246374654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:34,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:34,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246374654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:34,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:34,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:34,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:34,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:34,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:34,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:34,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:34,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246374663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:34,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:34,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:34,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:34,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:34,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246374673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:34,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:34,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:34,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:34,833 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/3e04cae0db914220becbae247176ae67 2024-12-03T17:18:34,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/2d948ec6cc1641a7b02fa50d41c77f29 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/2d948ec6cc1641a7b02fa50d41c77f29 2024-12-03T17:18:34,851 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/2d948ec6cc1641a7b02fa50d41c77f29, entries=150, sequenceid=206, filesize=30.4 K 2024-12-03T17:18:34,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/73a19e7e0edc4b6f9e4d5fe8e875388f as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/73a19e7e0edc4b6f9e4d5fe8e875388f 2024-12-03T17:18:34,856 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/73a19e7e0edc4b6f9e4d5fe8e875388f, entries=150, sequenceid=206, filesize=11.9 K 2024-12-03T17:18:34,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/3e04cae0db914220becbae247176ae67 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/3e04cae0db914220becbae247176ae67 2024-12-03T17:18:34,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/3e04cae0db914220becbae247176ae67, entries=150, sequenceid=206, filesize=11.9 K 2024-12-03T17:18:34,862 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 3e5f45a8f2579c6bf86ccea0cc76d7f9 in 1386ms, sequenceid=206, compaction requested=true 2024-12-03T17:18:34,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3e5f45a8f2579c6bf86ccea0cc76d7f9: 2024-12-03T17:18:34,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3e5f45a8f2579c6bf86ccea0cc76d7f9:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T17:18:34,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:18:34,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3e5f45a8f2579c6bf86ccea0cc76d7f9:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T17:18:34,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:18:34,862 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T17:18:34,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3e5f45a8f2579c6bf86ccea0cc76d7f9:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T17:18:34,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:18:34,862 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T17:18:34,864 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133406 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T17:18:34,864 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): 3e5f45a8f2579c6bf86ccea0cc76d7f9/A is initiating minor compaction (all files) 2024-12-03T17:18:34,864 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3e5f45a8f2579c6bf86ccea0cc76d7f9/A in TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:34,864 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/158f9f06ad7f4c3ba01a0f99522d4dc3, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/d0fc1619dcc74efb88cb697af66bc562, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/d099eda1d9d44fc68cce3682cc9c697a, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/2d948ec6cc1641a7b02fa50d41c77f29] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp, totalSize=130.3 K 2024-12-03T17:18:34,864 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:34,864 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. files: [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/158f9f06ad7f4c3ba01a0f99522d4dc3, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/d0fc1619dcc74efb88cb697af66bc562, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/d099eda1d9d44fc68cce3682cc9c697a, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/2d948ec6cc1641a7b02fa50d41c77f29] 2024-12-03T17:18:34,866 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T17:18:34,867 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): 3e5f45a8f2579c6bf86ccea0cc76d7f9/B is initiating minor compaction (all files) 2024-12-03T17:18:34,867 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3e5f45a8f2579c6bf86ccea0cc76d7f9/B in TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:34,867 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/7aa06e2d24ca4640b966889a17eb4dd9, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/29751bc810414d5d9c69e648609a86eb, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/8b31079b0a5c4202a43a68316da8d380, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/73a19e7e0edc4b6f9e4d5fe8e875388f] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp, totalSize=47.8 K 2024-12-03T17:18:34,867 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 158f9f06ad7f4c3ba01a0f99522d4dc3, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733246308783 2024-12-03T17:18:34,868 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0fc1619dcc74efb88cb697af66bc562, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733246309962 2024-12-03T17:18:34,868 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 7aa06e2d24ca4640b966889a17eb4dd9, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733246308783 2024-12-03T17:18:34,868 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 29751bc810414d5d9c69e648609a86eb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733246309969 2024-12-03T17:18:34,868 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting d099eda1d9d44fc68cce3682cc9c697a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733246311128 2024-12-03T17:18:34,869 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b31079b0a5c4202a43a68316da8d380, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733246311128 2024-12-03T17:18:34,869 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d948ec6cc1641a7b02fa50d41c77f29, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1733246313339 2024-12-03T17:18:34,869 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 73a19e7e0edc4b6f9e4d5fe8e875388f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1733246313339 2024-12-03T17:18:34,880 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3e5f45a8f2579c6bf86ccea0cc76d7f9#B#compaction#512 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:18:34,880 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/ca3dc74b0b014a8da08cb121d4465a31 is 50, key is test_row_0/B:col10/1733246313339/Put/seqid=0 2024-12-03T17:18:34,881 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3e5f45a8f2579c6bf86ccea0cc76d7f9] 2024-12-03T17:18:34,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742429_1605 (size=12629) 2024-12-03T17:18:34,899 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412036a02d8a903294e818b31b40281f6345b_3e5f45a8f2579c6bf86ccea0cc76d7f9 store=[table=TestAcidGuarantees family=A region=3e5f45a8f2579c6bf86ccea0cc76d7f9] 2024-12-03T17:18:34,901 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412036a02d8a903294e818b31b40281f6345b_3e5f45a8f2579c6bf86ccea0cc76d7f9, store=[table=TestAcidGuarantees family=A region=3e5f45a8f2579c6bf86ccea0cc76d7f9] 2024-12-03T17:18:34,902 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412036a02d8a903294e818b31b40281f6345b_3e5f45a8f2579c6bf86ccea0cc76d7f9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3e5f45a8f2579c6bf86ccea0cc76d7f9] 2024-12-03T17:18:34,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742430_1606 (size=4469) 2024-12-03T17:18:34,917 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3e5f45a8f2579c6bf86ccea0cc76d7f9#A#compaction#513 average throughput is 0.68 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:18:34,917 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/ec0437f3aab74089b184cc4eb4e046fe is 175, key is test_row_0/A:col10/1733246313339/Put/seqid=0 2024-12-03T17:18:34,918 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/ca3dc74b0b014a8da08cb121d4465a31 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/ca3dc74b0b014a8da08cb121d4465a31 2024-12-03T17:18:34,929 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3e5f45a8f2579c6bf86ccea0cc76d7f9/B of 3e5f45a8f2579c6bf86ccea0cc76d7f9 into ca3dc74b0b014a8da08cb121d4465a31(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:18:34,929 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3e5f45a8f2579c6bf86ccea0cc76d7f9: 2024-12-03T17:18:34,929 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., storeName=3e5f45a8f2579c6bf86ccea0cc76d7f9/B, priority=12, startTime=1733246314862; duration=0sec 2024-12-03T17:18:34,929 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:18:34,929 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3e5f45a8f2579c6bf86ccea0cc76d7f9:B 2024-12-03T17:18:34,929 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T17:18:34,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742431_1607 (size=31583) 2024-12-03T17:18:34,935 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T17:18:34,935 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): 3e5f45a8f2579c6bf86ccea0cc76d7f9/C is initiating minor compaction (all files) 2024-12-03T17:18:34,935 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3e5f45a8f2579c6bf86ccea0cc76d7f9/C in TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:34,935 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/5f68fdf67b8e4f9db4d917b44026058e, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/803719d3635947ccb29019f984071d54, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/d0786d5332d841bc942184191a12e3e6, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/3e04cae0db914220becbae247176ae67] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp, totalSize=47.8 K 2024-12-03T17:18:34,936 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f68fdf67b8e4f9db4d917b44026058e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733246308783 2024-12-03T17:18:34,936 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 803719d3635947ccb29019f984071d54, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733246309969 2024-12-03T17:18:34,937 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting d0786d5332d841bc942184191a12e3e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733246311128 2024-12-03T17:18:34,937 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e04cae0db914220becbae247176ae67, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1733246313339 2024-12-03T17:18:34,940 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/ec0437f3aab74089b184cc4eb4e046fe as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/ec0437f3aab74089b184cc4eb4e046fe 2024-12-03T17:18:34,950 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3e5f45a8f2579c6bf86ccea0cc76d7f9#C#compaction#514 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:18:34,950 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/7f901f1c117e43a9bcd7dd23c1b71a93 is 50, key is test_row_0/C:col10/1733246313339/Put/seqid=0 2024-12-03T17:18:34,951 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3e5f45a8f2579c6bf86ccea0cc76d7f9/A of 3e5f45a8f2579c6bf86ccea0cc76d7f9 into ec0437f3aab74089b184cc4eb4e046fe(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:18:34,951 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3e5f45a8f2579c6bf86ccea0cc76d7f9: 2024-12-03T17:18:34,951 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., storeName=3e5f45a8f2579c6bf86ccea0cc76d7f9/A, priority=12, startTime=1733246314862; duration=0sec 2024-12-03T17:18:34,951 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:18:34,951 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3e5f45a8f2579c6bf86ccea0cc76d7f9:A 2024-12-03T17:18:34,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742432_1608 (size=12629) 2024-12-03T17:18:35,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T17:18:35,067 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:18:35,068 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T17:18:35,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-12-03T17:18:35,075 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T17:18:35,077 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T17:18:35,077 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T17:18:35,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-03T17:18:35,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-03T17:18:35,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-03T17:18:35,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:35,234 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 3e5f45a8f2579c6bf86ccea0cc76d7f9 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-03T17:18:35,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e5f45a8f2579c6bf86ccea0cc76d7f9, store=A 2024-12-03T17:18:35,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:35,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e5f45a8f2579c6bf86ccea0cc76d7f9, store=B 2024-12-03T17:18:35,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:35,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e5f45a8f2579c6bf86ccea0cc76d7f9, store=C 2024-12-03T17:18:35,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:35,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412032a5b14bdfdc74c3bbbda00f5cc212ef4_3e5f45a8f2579c6bf86ccea0cc76d7f9 is 50, key is test_row_0/A:col10/1733246313503/Put/seqid=0 2024-12-03T17:18:35,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742433_1609 (size=12304) 2024-12-03T17:18:35,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-03T17:18:35,403 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/7f901f1c117e43a9bcd7dd23c1b71a93 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/7f901f1c117e43a9bcd7dd23c1b71a93 2024-12-03T17:18:35,413 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3e5f45a8f2579c6bf86ccea0cc76d7f9/C of 3e5f45a8f2579c6bf86ccea0cc76d7f9 into 7f901f1c117e43a9bcd7dd23c1b71a93(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:18:35,413 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3e5f45a8f2579c6bf86ccea0cc76d7f9: 2024-12-03T17:18:35,413 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., storeName=3e5f45a8f2579c6bf86ccea0cc76d7f9/C, priority=12, startTime=1733246314862; duration=0sec 2024-12-03T17:18:35,413 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:18:35,413 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3e5f45a8f2579c6bf86ccea0cc76d7f9:C 2024-12-03T17:18:35,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:18:35,663 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412032a5b14bdfdc74c3bbbda00f5cc212ef4_3e5f45a8f2579c6bf86ccea0cc76d7f9 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412032a5b14bdfdc74c3bbbda00f5cc212ef4_3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:35,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/714da794ca0c4f0c8bc6af8470ce96f0, store: [table=TestAcidGuarantees family=A region=3e5f45a8f2579c6bf86ccea0cc76d7f9] 2024-12-03T17:18:35,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/714da794ca0c4f0c8bc6af8470ce96f0 is 175, key is test_row_0/A:col10/1733246313503/Put/seqid=0 2024-12-03T17:18:35,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:35,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. as already flushing 2024-12-03T17:18:35,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742434_1610 (size=31105) 2024-12-03T17:18:35,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-03T17:18:35,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:35,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246375707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:35,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:35,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246375707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:35,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:35,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:35,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:35,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:35,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246375708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:35,713 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:35,713 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:35,713 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:35,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:35,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34552 deadline: 1733246375714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:35,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:35,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:35,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:35,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:35,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:35,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:35,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:35,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246375825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:35,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:35,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:35,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:35,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:35,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246375825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:35,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:35,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34552 deadline: 1733246375825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:35,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:35,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:35,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:35,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246375825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:35,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:35,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:35,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:35,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:35,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:35,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:35,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:36,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:36,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246376033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:36,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:36,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246376034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:36,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:36,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:36,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:36,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:36,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:36,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:36,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:36,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34552 deadline: 1733246376034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:36,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:36,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:36,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:36,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:36,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246376034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:36,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:36,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:36,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:36,099 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=226, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/714da794ca0c4f0c8bc6af8470ce96f0 2024-12-03T17:18:36,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/4b67165ebdee4ce5af9113e8541bf7d0 is 50, key is test_row_0/B:col10/1733246313503/Put/seqid=0 2024-12-03T17:18:36,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742435_1611 (size=12151) 2024-12-03T17:18:36,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-03T17:18:36,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:36,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246376343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:36,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:36,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246376344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:36,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:36,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:36,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:36,346 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:36,346 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:36,346 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:36,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:36,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34552 deadline: 1733246376344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:36,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:36,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:36,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246376344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:36,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:36,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:36,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:36,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:36,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:36,603 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/4b67165ebdee4ce5af9113e8541bf7d0 2024-12-03T17:18:36,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/f7aa2098da7649d69890dc5b1b395ec8 is 50, key is test_row_0/C:col10/1733246313503/Put/seqid=0 2024-12-03T17:18:36,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742436_1612 (size=12151) 2024-12-03T17:18:36,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:36,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246376855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:36,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:36,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:36,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:36,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:36,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246376855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:36,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:36,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:36,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:36,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:36,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246376855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:36,857 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:36,857 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:36,857 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:36,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:36,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34552 deadline: 1733246376859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:36,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:36,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:36,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:37,046 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/f7aa2098da7649d69890dc5b1b395ec8 2024-12-03T17:18:37,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/714da794ca0c4f0c8bc6af8470ce96f0 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/714da794ca0c4f0c8bc6af8470ce96f0 2024-12-03T17:18:37,055 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/714da794ca0c4f0c8bc6af8470ce96f0, entries=150, sequenceid=226, filesize=30.4 K 2024-12-03T17:18:37,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/4b67165ebdee4ce5af9113e8541bf7d0 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/4b67165ebdee4ce5af9113e8541bf7d0 2024-12-03T17:18:37,059 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/4b67165ebdee4ce5af9113e8541bf7d0, entries=150, sequenceid=226, filesize=11.9 K 2024-12-03T17:18:37,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/f7aa2098da7649d69890dc5b1b395ec8 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/f7aa2098da7649d69890dc5b1b395ec8 2024-12-03T17:18:37,063 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/f7aa2098da7649d69890dc5b1b395ec8, entries=150, sequenceid=226, filesize=11.9 K 2024-12-03T17:18:37,064 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 3e5f45a8f2579c6bf86ccea0cc76d7f9 in 1830ms, sequenceid=226, compaction requested=false 2024-12-03T17:18:37,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 3e5f45a8f2579c6bf86ccea0cc76d7f9: 2024-12-03T17:18:37,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:37,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-12-03T17:18:37,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-12-03T17:18:37,071 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-12-03T17:18:37,071 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9890 sec 2024-12-03T17:18:37,073 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 2.0030 sec 2024-12-03T17:18:37,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-03T17:18:37,224 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:18:37,224 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T17:18:37,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-12-03T17:18:37,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-03T17:18:37,231 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T17:18:37,232 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T17:18:37,232 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T17:18:37,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-03T17:18:37,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-03T17:18:37,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:37,385 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 3e5f45a8f2579c6bf86ccea0cc76d7f9 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-03T17:18:37,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e5f45a8f2579c6bf86ccea0cc76d7f9, store=A 2024-12-03T17:18:37,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:37,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e5f45a8f2579c6bf86ccea0cc76d7f9, store=B 2024-12-03T17:18:37,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:37,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e5f45a8f2579c6bf86ccea0cc76d7f9, store=C 2024-12-03T17:18:37,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:37,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203719170cf92df4499a9dc8ce1150a3df8_3e5f45a8f2579c6bf86ccea0cc76d7f9 is 50, key is test_row_0/A:col10/1733246315706/Put/seqid=0 2024-12-03T17:18:37,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742437_1613 (size=12304) 2024-12-03T17:18:37,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-03T17:18:37,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:18:37,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-03T17:18:37,867 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203719170cf92df4499a9dc8ce1150a3df8_3e5f45a8f2579c6bf86ccea0cc76d7f9 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203719170cf92df4499a9dc8ce1150a3df8_3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:37,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:37,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/10089801d0c34e3bb4e12349bd9dfdf9, store: [table=TestAcidGuarantees family=A region=3e5f45a8f2579c6bf86ccea0cc76d7f9] 2024-12-03T17:18:37,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. as already flushing 2024-12-03T17:18:37,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/10089801d0c34e3bb4e12349bd9dfdf9 is 175, key is test_row_0/A:col10/1733246315706/Put/seqid=0 2024-12-03T17:18:37,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742438_1614 (size=31105) 2024-12-03T17:18:37,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:37,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246377900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:37,908 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:37,908 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:37,908 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:37,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:37,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246377904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:37,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:37,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:37,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:37,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:37,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34552 deadline: 1733246377908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:37,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:37,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246377908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:37,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:37,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:37,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:37,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:37,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:37,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:38,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:38,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246378014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:38,018 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:38,018 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:38,018 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:38,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:38,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34552 deadline: 1733246378024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:38,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:38,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:38,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246378024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:38,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:38,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:38,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:38,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:38,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:38,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:38,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246378024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:38,030 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:38,030 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:38,030 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:38,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:38,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34634 deadline: 1733246378203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:38,208 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 27e0f852a164,41167,1733246151593 for 'test_row_2' in 3e5f45a8f2579c6bf86ccea0cc76d7f9 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8231 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T17:18:38,209 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:38,209 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:38,209 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:38,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:38,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246378224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:38,229 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:38,229 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:38,229 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:38,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:38,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34552 deadline: 1733246378233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:38,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:38,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:38,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:38,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246378233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:38,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:38,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:38,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:38,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:38,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:38,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246378233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:38,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:38,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:38,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:38,287 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=241, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/10089801d0c34e3bb4e12349bd9dfdf9 2024-12-03T17:18:38,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/f33ac27fc61c4feaac7754c0145f1759 is 50, key is test_row_0/B:col10/1733246315706/Put/seqid=0 2024-12-03T17:18:38,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742439_1615 (size=12151) 2024-12-03T17:18:38,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-03T17:18:38,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:38,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246378534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:38,537 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:38,537 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:38,537 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:38,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:38,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246378543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:38,550 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:38,550 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:38,550 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:38,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:38,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34552 deadline: 1733246378547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:38,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:38,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:38,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:38,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:38,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246378547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:38,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:38,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:38,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:38,715 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/f33ac27fc61c4feaac7754c0145f1759 2024-12-03T17:18:38,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/597e352b7c1e44d5b887a9e0e8fd4f9e is 50, key is test_row_0/C:col10/1733246315706/Put/seqid=0 2024-12-03T17:18:38,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742440_1616 (size=12151) 2024-12-03T17:18:38,743 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/597e352b7c1e44d5b887a9e0e8fd4f9e 2024-12-03T17:18:38,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/10089801d0c34e3bb4e12349bd9dfdf9 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/10089801d0c34e3bb4e12349bd9dfdf9 2024-12-03T17:18:38,754 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/10089801d0c34e3bb4e12349bd9dfdf9, entries=150, sequenceid=241, filesize=30.4 K 2024-12-03T17:18:38,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/f33ac27fc61c4feaac7754c0145f1759 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/f33ac27fc61c4feaac7754c0145f1759 2024-12-03T17:18:38,759 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/f33ac27fc61c4feaac7754c0145f1759, entries=150, sequenceid=241, filesize=11.9 K 2024-12-03T17:18:38,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/597e352b7c1e44d5b887a9e0e8fd4f9e as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/597e352b7c1e44d5b887a9e0e8fd4f9e 2024-12-03T17:18:38,764 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/597e352b7c1e44d5b887a9e0e8fd4f9e, entries=150, sequenceid=241, filesize=11.9 K 2024-12-03T17:18:38,765 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 3e5f45a8f2579c6bf86ccea0cc76d7f9 in 1380ms, sequenceid=241, compaction requested=true 2024-12-03T17:18:38,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 3e5f45a8f2579c6bf86ccea0cc76d7f9: 2024-12-03T17:18:38,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:38,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-12-03T17:18:38,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-12-03T17:18:38,769 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-12-03T17:18:38,769 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5350 sec 2024-12-03T17:18:38,771 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.5440 sec 2024-12-03T17:18:39,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:39,053 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3e5f45a8f2579c6bf86ccea0cc76d7f9 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-03T17:18:39,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e5f45a8f2579c6bf86ccea0cc76d7f9, store=A 2024-12-03T17:18:39,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:39,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e5f45a8f2579c6bf86ccea0cc76d7f9, store=B 2024-12-03T17:18:39,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:39,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e5f45a8f2579c6bf86ccea0cc76d7f9, store=C 2024-12-03T17:18:39,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:39,069 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203d6537a7e7f144e1d887ee934b396d6ad_3e5f45a8f2579c6bf86ccea0cc76d7f9 is 50, key is test_row_0/A:col10/1733246319047/Put/seqid=0 2024-12-03T17:18:39,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:39,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246379076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:39,087 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:39,087 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:39,087 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:39,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:39,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34552 deadline: 1733246379083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:39,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:39,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:39,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:39,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:39,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246379087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:39,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:39,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:39,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:39,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742441_1617 (size=12404) 2024-12-03T17:18:39,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:39,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246379094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:39,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:39,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:39,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:39,102 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:18:39,118 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203d6537a7e7f144e1d887ee934b396d6ad_3e5f45a8f2579c6bf86ccea0cc76d7f9 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d6537a7e7f144e1d887ee934b396d6ad_3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:39,121 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/ca14b32cbc16425481f860c80947712e, store: [table=TestAcidGuarantees family=A region=3e5f45a8f2579c6bf86ccea0cc76d7f9] 2024-12-03T17:18:39,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/ca14b32cbc16425481f860c80947712e is 175, key is test_row_0/A:col10/1733246319047/Put/seqid=0 2024-12-03T17:18:39,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742442_1618 (size=31205) 2024-12-03T17:18:39,138 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=260, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/ca14b32cbc16425481f860c80947712e 2024-12-03T17:18:39,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/48ae7a1f49b9436eafb7e2e4623a86f7 is 50, key is test_row_0/B:col10/1733246319047/Put/seqid=0 2024-12-03T17:18:39,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:39,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246379194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:39,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:39,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34552 deadline: 1733246379194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:39,201 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:39,201 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:39,201 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:39,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:39,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:39,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:39,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742443_1619 (size=12251) 2024-12-03T17:18:39,207 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/48ae7a1f49b9436eafb7e2e4623a86f7 2024-12-03T17:18:39,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:39,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246379204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:39,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:39,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246379207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:39,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:39,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:39,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:39,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:39,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:39,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:39,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/0f960c382d0c47e7bd4024bb71eb7b70 is 50, key is test_row_0/C:col10/1733246319047/Put/seqid=0 2024-12-03T17:18:39,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742444_1620 (size=12251) 2024-12-03T17:18:39,248 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/0f960c382d0c47e7bd4024bb71eb7b70 2024-12-03T17:18:39,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/ca14b32cbc16425481f860c80947712e as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/ca14b32cbc16425481f860c80947712e 2024-12-03T17:18:39,259 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/ca14b32cbc16425481f860c80947712e, entries=150, sequenceid=260, filesize=30.5 K 2024-12-03T17:18:39,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/48ae7a1f49b9436eafb7e2e4623a86f7 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/48ae7a1f49b9436eafb7e2e4623a86f7 2024-12-03T17:18:39,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/48ae7a1f49b9436eafb7e2e4623a86f7, entries=150, sequenceid=260, filesize=12.0 K 2024-12-03T17:18:39,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/0f960c382d0c47e7bd4024bb71eb7b70 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/0f960c382d0c47e7bd4024bb71eb7b70 2024-12-03T17:18:39,268 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/0f960c382d0c47e7bd4024bb71eb7b70, entries=150, sequenceid=260, filesize=12.0 K 2024-12-03T17:18:39,269 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 3e5f45a8f2579c6bf86ccea0cc76d7f9 in 215ms, sequenceid=260, compaction requested=true 2024-12-03T17:18:39,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3e5f45a8f2579c6bf86ccea0cc76d7f9: 2024-12-03T17:18:39,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3e5f45a8f2579c6bf86ccea0cc76d7f9:A, priority=-2147483648, current under compaction store size is 1 2024-12-03T17:18:39,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:18:39,269 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T17:18:39,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3e5f45a8f2579c6bf86ccea0cc76d7f9:B, priority=-2147483648, current under compaction store size is 2 2024-12-03T17:18:39,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:18:39,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3e5f45a8f2579c6bf86ccea0cc76d7f9:C, priority=-2147483648, current under compaction store size is 3 2024-12-03T17:18:39,269 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T17:18:39,270 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 124998 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T17:18:39,271 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1541): 3e5f45a8f2579c6bf86ccea0cc76d7f9/A is initiating minor compaction (all files) 2024-12-03T17:18:39,271 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3e5f45a8f2579c6bf86ccea0cc76d7f9/A in TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:39,271 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/ec0437f3aab74089b184cc4eb4e046fe, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/714da794ca0c4f0c8bc6af8470ce96f0, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/10089801d0c34e3bb4e12349bd9dfdf9, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/ca14b32cbc16425481f860c80947712e] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp, totalSize=122.1 K 2024-12-03T17:18:39,271 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:39,271 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. files: [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/ec0437f3aab74089b184cc4eb4e046fe, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/714da794ca0c4f0c8bc6af8470ce96f0, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/10089801d0c34e3bb4e12349bd9dfdf9, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/ca14b32cbc16425481f860c80947712e] 2024-12-03T17:18:39,271 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49182 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T17:18:39,271 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec0437f3aab74089b184cc4eb4e046fe, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1733246313339 2024-12-03T17:18:39,271 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): 3e5f45a8f2579c6bf86ccea0cc76d7f9/B is initiating minor compaction (all files) 2024-12-03T17:18:39,272 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3e5f45a8f2579c6bf86ccea0cc76d7f9/B in TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:39,272 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/ca3dc74b0b014a8da08cb121d4465a31, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/4b67165ebdee4ce5af9113e8541bf7d0, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/f33ac27fc61c4feaac7754c0145f1759, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/48ae7a1f49b9436eafb7e2e4623a86f7] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp, totalSize=48.0 K 2024-12-03T17:18:39,272 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 714da794ca0c4f0c8bc6af8470ce96f0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1733246313503 2024-12-03T17:18:39,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:18:39,272 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting ca3dc74b0b014a8da08cb121d4465a31, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1733246313339 2024-12-03T17:18:39,272 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b67165ebdee4ce5af9113e8541bf7d0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1733246313503 2024-12-03T17:18:39,272 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10089801d0c34e3bb4e12349bd9dfdf9, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733246315704 2024-12-03T17:18:39,272 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting f33ac27fc61c4feaac7754c0145f1759, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733246315704 2024-12-03T17:18:39,272 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca14b32cbc16425481f860c80947712e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1733246317902 2024-12-03T17:18:39,273 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 48ae7a1f49b9436eafb7e2e4623a86f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1733246317902 2024-12-03T17:18:39,292 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3e5f45a8f2579c6bf86ccea0cc76d7f9] 2024-12-03T17:18:39,296 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3e5f45a8f2579c6bf86ccea0cc76d7f9#B#compaction#525 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:18:39,296 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/be63c862379f41969bbbc8106f244141 is 50, key is test_row_0/B:col10/1733246319047/Put/seqid=0 2024-12-03T17:18:39,299 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412035a59c3c7a9e040edad296e1cf7c9c240_3e5f45a8f2579c6bf86ccea0cc76d7f9 store=[table=TestAcidGuarantees family=A region=3e5f45a8f2579c6bf86ccea0cc76d7f9] 2024-12-03T17:18:39,302 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412035a59c3c7a9e040edad296e1cf7c9c240_3e5f45a8f2579c6bf86ccea0cc76d7f9, store=[table=TestAcidGuarantees family=A region=3e5f45a8f2579c6bf86ccea0cc76d7f9] 2024-12-03T17:18:39,303 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412035a59c3c7a9e040edad296e1cf7c9c240_3e5f45a8f2579c6bf86ccea0cc76d7f9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3e5f45a8f2579c6bf86ccea0cc76d7f9] 2024-12-03T17:18:39,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742445_1621 (size=12865) 2024-12-03T17:18:39,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742446_1622 (size=4469) 2024-12-03T17:18:39,334 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/be63c862379f41969bbbc8106f244141 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/be63c862379f41969bbbc8106f244141 2024-12-03T17:18:39,355 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3e5f45a8f2579c6bf86ccea0cc76d7f9/B of 3e5f45a8f2579c6bf86ccea0cc76d7f9 into be63c862379f41969bbbc8106f244141(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:18:39,355 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3e5f45a8f2579c6bf86ccea0cc76d7f9: 2024-12-03T17:18:39,355 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., storeName=3e5f45a8f2579c6bf86ccea0cc76d7f9/B, priority=12, startTime=1733246319269; duration=0sec 2024-12-03T17:18:39,355 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-03T17:18:39,355 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3e5f45a8f2579c6bf86ccea0cc76d7f9:B 2024-12-03T17:18:39,356 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T17:18:39,359 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49182 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-03T17:18:39,360 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1541): 3e5f45a8f2579c6bf86ccea0cc76d7f9/C is initiating minor compaction (all files) 2024-12-03T17:18:39,360 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3e5f45a8f2579c6bf86ccea0cc76d7f9/C in TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:39,360 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/7f901f1c117e43a9bcd7dd23c1b71a93, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/f7aa2098da7649d69890dc5b1b395ec8, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/597e352b7c1e44d5b887a9e0e8fd4f9e, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/0f960c382d0c47e7bd4024bb71eb7b70] into tmpdir=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp, totalSize=48.0 K 2024-12-03T17:18:39,360 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f901f1c117e43a9bcd7dd23c1b71a93, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1733246313339 2024-12-03T17:18:39,360 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting f7aa2098da7649d69890dc5b1b395ec8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1733246313503 2024-12-03T17:18:39,361 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 597e352b7c1e44d5b887a9e0e8fd4f9e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1733246315704 2024-12-03T17:18:39,361 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f960c382d0c47e7bd4024bb71eb7b70, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1733246317902 2024-12-03T17:18:39,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-03T17:18:39,384 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:18:39,384 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-03T17:18:39,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-12-03T17:18:39,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-03T17:18:39,391 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-03T17:18:39,392 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-03T17:18:39,392 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-03T17:18:39,393 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3e5f45a8f2579c6bf86ccea0cc76d7f9#C#compaction#526 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:18:39,394 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/46b8b4f3971c4cb8a2d1e2ed8dfe737c is 50, key is test_row_0/C:col10/1733246319047/Put/seqid=0 2024-12-03T17:18:39,409 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3e5f45a8f2579c6bf86ccea0cc76d7f9 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-03T17:18:39,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e5f45a8f2579c6bf86ccea0cc76d7f9, store=A 2024-12-03T17:18:39,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:39,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e5f45a8f2579c6bf86ccea0cc76d7f9, store=B 2024-12-03T17:18:39,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:39,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e5f45a8f2579c6bf86ccea0cc76d7f9, store=C 2024-12-03T17:18:39,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:39,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742447_1623 (size=12865) 2024-12-03T17:18:39,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:39,440 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203304d189c5e544b6cba509893a2be9e8a_3e5f45a8f2579c6bf86ccea0cc76d7f9 is 50, key is test_row_0/A:col10/1733246319082/Put/seqid=0 2024-12-03T17:18:39,447 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/46b8b4f3971c4cb8a2d1e2ed8dfe737c as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/46b8b4f3971c4cb8a2d1e2ed8dfe737c 2024-12-03T17:18:39,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742448_1624 (size=14994) 2024-12-03T17:18:39,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:39,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246379450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:39,460 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:39,460 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:39,460 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:39,464 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3e5f45a8f2579c6bf86ccea0cc76d7f9/C of 3e5f45a8f2579c6bf86ccea0cc76d7f9 into 46b8b4f3971c4cb8a2d1e2ed8dfe737c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:18:39,464 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3e5f45a8f2579c6bf86ccea0cc76d7f9: 2024-12-03T17:18:39,465 INFO [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., storeName=3e5f45a8f2579c6bf86ccea0cc76d7f9/C, priority=12, startTime=1733246319269; duration=0sec 2024-12-03T17:18:39,465 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:18:39,465 DEBUG [RS:0;27e0f852a164:41167-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3e5f45a8f2579c6bf86ccea0cc76d7f9:C 2024-12-03T17:18:39,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:39,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34552 deadline: 1733246379459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:39,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:39,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:39,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:39,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:39,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246379461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:39,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:39,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:39,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:39,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:39,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246379463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:39,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:39,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:39,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:39,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-03T17:18:39,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-03T17:18:39,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:39,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. as already flushing 2024-12-03T17:18:39,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:39,547 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:39,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:39,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:39,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:39,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246379563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:39,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:39,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:39,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:39,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:39,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:39,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246379574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:39,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34552 deadline: 1733246379574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:39,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:39,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:39,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:39,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246379574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:39,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:39,582 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:39,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:39,582 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:39,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:39,582 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:39,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:39,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-03T17:18:39,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:39,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. as already flushing 2024-12-03T17:18:39,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:39,703 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:39,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:39,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:39,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-03T17:18:39,735 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3e5f45a8f2579c6bf86ccea0cc76d7f9#A#compaction#524 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T17:18:39,736 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/9d43a76483a24ef18f188e70c03605ea is 175, key is test_row_0/A:col10/1733246319047/Put/seqid=0 2024-12-03T17:18:39,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:39,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246379773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:39,780 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:39,780 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:39,780 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:39,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742449_1625 (size=31819) 2024-12-03T17:18:39,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:39,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246379784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:39,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:39,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:39,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:39,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:39,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246379793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:39,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:39,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:39,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:39,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:39,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34552 deadline: 1733246379793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:39,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:39,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:39,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:39,803 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/9d43a76483a24ef18f188e70c03605ea as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/9d43a76483a24ef18f188e70c03605ea 2024-12-03T17:18:39,819 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3e5f45a8f2579c6bf86ccea0cc76d7f9/A of 3e5f45a8f2579c6bf86ccea0cc76d7f9 into 9d43a76483a24ef18f188e70c03605ea(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T17:18:39,819 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3e5f45a8f2579c6bf86ccea0cc76d7f9: 2024-12-03T17:18:39,820 INFO [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., storeName=3e5f45a8f2579c6bf86ccea0cc76d7f9/A, priority=12, startTime=1733246319269; duration=0sec 2024-12-03T17:18:39,820 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-03T17:18:39,820 DEBUG [RS:0;27e0f852a164:41167-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3e5f45a8f2579c6bf86ccea0cc76d7f9:A 2024-12-03T17:18:39,853 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:18:39,863 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241203304d189c5e544b6cba509893a2be9e8a_3e5f45a8f2579c6bf86ccea0cc76d7f9 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203304d189c5e544b6cba509893a2be9e8a_3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:39,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-03T17:18:39,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:39,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. as already flushing 2024-12-03T17:18:39,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:39,866 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/e557935fcbfa46ff95bc67aabd473280, store: [table=TestAcidGuarantees family=A region=3e5f45a8f2579c6bf86ccea0cc76d7f9] 2024-12-03T17:18:39,866 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:39,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:39,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/e557935fcbfa46ff95bc67aabd473280 is 175, key is test_row_0/A:col10/1733246319082/Put/seqid=0 2024-12-03T17:18:39,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:39,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742450_1626 (size=39949) 2024-12-03T17:18:39,918 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=276, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/e557935fcbfa46ff95bc67aabd473280 2024-12-03T17:18:39,946 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/39f8d67dd48e40ba8ca6bad4120b5441 is 50, key is test_row_0/B:col10/1733246319082/Put/seqid=0 2024-12-03T17:18:39,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742451_1627 (size=12301) 2024-12-03T17:18:40,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-03T17:18:40,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:40,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. as already flushing 2024-12-03T17:18:40,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:40,027 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:40,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:40,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-03T17:18:40,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:40,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:40,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34602 deadline: 1733246380084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:40,091 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:40,091 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:40,091 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:40,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:40,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34584 deadline: 1733246380093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:40,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:40,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:40,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:40,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:40,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34586 deadline: 1733246380103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:40,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:40,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:40,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:40,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T17:18:40,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34552 deadline: 1733246380114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 2024-12-03T17:18:40,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 , the old value is region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:40,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T17:18:40,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., hostname=27e0f852a164,41167,1733246151593, seqNum=5 because the exception is null or not the one we care about 2024-12-03T17:18:40,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-03T17:18:40,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:40,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. as already flushing 2024-12-03T17:18:40,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:40,187 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:40,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:40,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:40,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-03T17:18:40,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:40,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. as already flushing 2024-12-03T17:18:40,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:40,347 ERROR [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:40,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:40,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T17:18:40,380 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/39f8d67dd48e40ba8ca6bad4120b5441 2024-12-03T17:18:40,392 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/52405f55fdeb4c5190423fd27ee8a366 is 50, key is test_row_0/C:col10/1733246319082/Put/seqid=0 2024-12-03T17:18:40,400 INFO [Thread-2453 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2453. 2024-12-03T17:18:40,400 DEBUG [Thread-2453 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:18:40,400 DEBUG [Thread-2453 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:40,400 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:18:40,400 DEBUG [Thread-2453 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:40,402 INFO [Thread-2451 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2451. 2024-12-03T17:18:40,402 DEBUG [Thread-2451 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:18:40,402 DEBUG [Thread-2451 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:40,402 DEBUG [Thread-2451 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:40,404 INFO [Thread-2449 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2449. 2024-12-03T17:18:40,404 DEBUG [Thread-2449 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:18:40,404 DEBUG [Thread-2449 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:40,404 DEBUG [Thread-2449 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:40,404 INFO [Thread-2455 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2455. 2024-12-03T17:18:40,404 DEBUG [Thread-2455 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:18:40,405 DEBUG [Thread-2455 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:40,405 DEBUG [Thread-2455 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:40,405 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:18:40,405 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:18:40,406 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:18:40,407 INFO [Thread-2457 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2457. 2024-12-03T17:18:40,407 DEBUG [Thread-2457 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:18:40,407 DEBUG [Thread-2457 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:40,407 DEBUG [Thread-2457 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:40,407 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:18:40,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742452_1628 (size=12301) 2024-12-03T17:18:40,412 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/52405f55fdeb4c5190423fd27ee8a366 2024-12-03T17:18:40,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/e557935fcbfa46ff95bc67aabd473280 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/e557935fcbfa46ff95bc67aabd473280 2024-12-03T17:18:40,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/e557935fcbfa46ff95bc67aabd473280, entries=200, sequenceid=276, filesize=39.0 K 2024-12-03T17:18:40,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/39f8d67dd48e40ba8ca6bad4120b5441 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/39f8d67dd48e40ba8ca6bad4120b5441 2024-12-03T17:18:40,427 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/39f8d67dd48e40ba8ca6bad4120b5441, entries=150, sequenceid=276, filesize=12.0 K 2024-12-03T17:18:40,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/52405f55fdeb4c5190423fd27ee8a366 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/52405f55fdeb4c5190423fd27ee8a366 2024-12-03T17:18:40,430 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/52405f55fdeb4c5190423fd27ee8a366, entries=150, sequenceid=276, filesize=12.0 K 2024-12-03T17:18:40,434 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 3e5f45a8f2579c6bf86ccea0cc76d7f9 in 1026ms, sequenceid=276, compaction requested=false 2024-12-03T17:18:40,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3e5f45a8f2579c6bf86ccea0cc76d7f9: 2024-12-03T17:18:40,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41167 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-03T17:18:40,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:40,502 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 3e5f45a8f2579c6bf86ccea0cc76d7f9 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-03T17:18:40,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e5f45a8f2579c6bf86ccea0cc76d7f9, store=A 2024-12-03T17:18:40,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:40,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e5f45a8f2579c6bf86ccea0cc76d7f9, store=B 2024-12-03T17:18:40,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:40,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e5f45a8f2579c6bf86ccea0cc76d7f9, store=C 2024-12-03T17:18:40,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:40,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412030acec4eafc4945a9b4d9a7f96bfd678e_3e5f45a8f2579c6bf86ccea0cc76d7f9 is 50, key is test_row_0/A:col10/1733246319437/Put/seqid=0 2024-12-03T17:18:40,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742453_1629 (size=12454) 2024-12-03T17:18:40,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:18:40,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-03T17:18:40,539 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412030acec4eafc4945a9b4d9a7f96bfd678e_3e5f45a8f2579c6bf86ccea0cc76d7f9 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412030acec4eafc4945a9b4d9a7f96bfd678e_3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:40,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/4b805274fdba4d4ea0b2e385d7f9b9a9, store: [table=TestAcidGuarantees family=A region=3e5f45a8f2579c6bf86ccea0cc76d7f9] 2024-12-03T17:18:40,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/4b805274fdba4d4ea0b2e385d7f9b9a9 is 175, key is test_row_0/A:col10/1733246319437/Put/seqid=0 2024-12-03T17:18:40,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742454_1630 (size=31255) 2024-12-03T17:18:40,551 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=296, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/4b805274fdba4d4ea0b2e385d7f9b9a9 2024-12-03T17:18:40,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/1a87d4a459354a32a6ed4ff83a6e1c3d is 50, key is test_row_0/B:col10/1733246319437/Put/seqid=0 2024-12-03T17:18:40,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742455_1631 (size=12301) 2024-12-03T17:18:40,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41167 {}] regionserver.HRegion(8855): Flush requested on 3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:40,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. as already flushing 2024-12-03T17:18:40,605 INFO [Thread-2444 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2444. 2024-12-03T17:18:40,605 DEBUG [Thread-2444 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:18:40,605 DEBUG [Thread-2444 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:40,605 DEBUG [Thread-2444 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:40,605 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:18:40,615 INFO [Thread-2442 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2442. 2024-12-03T17:18:40,615 INFO [Thread-2438 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2438. 2024-12-03T17:18:40,615 DEBUG [Thread-2442 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:18:40,615 DEBUG [Thread-2438 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:18:40,615 DEBUG [Thread-2438 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:40,615 DEBUG [Thread-2442 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:40,615 DEBUG [Thread-2442 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:40,615 DEBUG [Thread-2438 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:40,615 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:18:40,615 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:18:40,635 INFO [Thread-2446 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2446. 2024-12-03T17:18:40,635 DEBUG [Thread-2446 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:18:40,635 DEBUG [Thread-2446 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:40,636 DEBUG [Thread-2446 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:40,636 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:18:40,960 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/1a87d4a459354a32a6ed4ff83a6e1c3d 2024-12-03T17:18:40,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/8dcb6ed9a48c4b5ab65ea4beb816ad99 is 50, key is test_row_0/C:col10/1733246319437/Put/seqid=0 2024-12-03T17:18:40,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742456_1632 (size=12301) 2024-12-03T17:18:40,969 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/8dcb6ed9a48c4b5ab65ea4beb816ad99 2024-12-03T17:18:40,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/4b805274fdba4d4ea0b2e385d7f9b9a9 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/4b805274fdba4d4ea0b2e385d7f9b9a9 2024-12-03T17:18:40,975 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/4b805274fdba4d4ea0b2e385d7f9b9a9, entries=150, sequenceid=296, filesize=30.5 K 2024-12-03T17:18:40,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/1a87d4a459354a32a6ed4ff83a6e1c3d as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/1a87d4a459354a32a6ed4ff83a6e1c3d 2024-12-03T17:18:40,979 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/1a87d4a459354a32a6ed4ff83a6e1c3d, entries=150, sequenceid=296, filesize=12.0 K 2024-12-03T17:18:40,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/8dcb6ed9a48c4b5ab65ea4beb816ad99 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/8dcb6ed9a48c4b5ab65ea4beb816ad99 2024-12-03T17:18:40,983 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/8dcb6ed9a48c4b5ab65ea4beb816ad99, entries=150, sequenceid=296, filesize=12.0 K 2024-12-03T17:18:40,983 INFO [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=26.84 KB/27480 for 3e5f45a8f2579c6bf86ccea0cc76d7f9 in 481ms, sequenceid=296, compaction requested=true 2024-12-03T17:18:40,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 3e5f45a8f2579c6bf86ccea0cc76d7f9: 2024-12-03T17:18:40,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:40,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/27e0f852a164:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-12-03T17:18:40,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-12-03T17:18:40,987 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-12-03T17:18:40,987 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5930 sec 2024-12-03T17:18:40,988 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.6030 sec 2024-12-03T17:18:41,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-03T17:18:41,544 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:18:48,306 INFO [Thread-2440 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2440. 2024-12-03T17:18:48,306 DEBUG [Thread-2440 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-03T17:18:48,306 DEBUG [Thread-2440 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:48,307 DEBUG [Thread-2440 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:48,307 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:18:48,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-03T17:18:48,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-12-03T17:18:48,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 9 2024-12-03T17:18:48,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-12-03T17:18:48,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-12-03T17:18:48,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 74 2024-12-03T17:18:48,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-03T17:18:48,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2202 2024-12-03T17:18:48,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2241 2024-12-03T17:18:48,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2131 2024-12-03T17:18:48,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2222 2024-12-03T17:18:48,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2258 2024-12-03T17:18:48,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-03T17:18:48,307 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T17:18:48,307 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T17:18:48,307 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:48,307 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:48,308 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:18:48,310 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-03T17:18:48,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-03T17:18:48,323 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733246328322"}]},"ts":"1733246328322"} 2024-12-03T17:18:48,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-03T17:18:48,324 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-03T17:18:48,324 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-03T17:18:48,326 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-03T17:18:48,328 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=158, ppid=157, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3e5f45a8f2579c6bf86ccea0cc76d7f9, UNASSIGN}] 2024-12-03T17:18:48,329 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=158, ppid=157, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3e5f45a8f2579c6bf86ccea0cc76d7f9, UNASSIGN 2024-12-03T17:18:48,330 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=158 updating hbase:meta row=3e5f45a8f2579c6bf86ccea0cc76d7f9, regionState=CLOSING, regionLocation=27e0f852a164,41167,1733246151593 2024-12-03T17:18:48,335 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=158, ppid=157, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3e5f45a8f2579c6bf86ccea0cc76d7f9, UNASSIGN because future has completed 2024-12-03T17:18:48,335 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T17:18:48,335 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593}] 2024-12-03T17:18:48,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-03T17:18:48,488 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] handler.UnassignRegionHandler(122): Close 3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:48,488 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T17:18:48,488 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1722): Closing 3e5f45a8f2579c6bf86ccea0cc76d7f9, disabling compactions & flushes 2024-12-03T17:18:48,488 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:48,488 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:48,488 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. after waiting 0 ms 2024-12-03T17:18:48,488 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:48,488 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(2902): Flushing 3e5f45a8f2579c6bf86ccea0cc76d7f9 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-03T17:18:48,489 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e5f45a8f2579c6bf86ccea0cc76d7f9, store=A 2024-12-03T17:18:48,489 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:48,489 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e5f45a8f2579c6bf86ccea0cc76d7f9, store=B 2024-12-03T17:18:48,489 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:48,489 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3e5f45a8f2579c6bf86ccea0cc76d7f9, store=C 2024-12-03T17:18:48,489 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-03T17:18:48,514 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412036f512a0b1e854af4a4480e736d59b7a4_3e5f45a8f2579c6bf86ccea0cc76d7f9 is 50, key is test_row_0/A:col10/1733246320614/Put/seqid=0 2024-12-03T17:18:48,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742457_1633 (size=12454) 2024-12-03T17:18:48,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-03T17:18:48,951 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T17:18:48,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-03T17:18:48,957 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412036f512a0b1e854af4a4480e736d59b7a4_3e5f45a8f2579c6bf86ccea0cc76d7f9 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036f512a0b1e854af4a4480e736d59b7a4_3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:48,958 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/55324c50fbfc42838b819a9d45a8ec72, store: [table=TestAcidGuarantees family=A region=3e5f45a8f2579c6bf86ccea0cc76d7f9] 2024-12-03T17:18:48,959 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/55324c50fbfc42838b819a9d45a8ec72 is 175, key is test_row_0/A:col10/1733246320614/Put/seqid=0 2024-12-03T17:18:48,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742458_1634 (size=31255) 2024-12-03T17:18:49,371 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=304, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/55324c50fbfc42838b819a9d45a8ec72 2024-12-03T17:18:49,379 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/f4434a9c6ae3424cad3d8adb62ce708a is 50, key is test_row_0/B:col10/1733246320614/Put/seqid=0 2024-12-03T17:18:49,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742459_1635 (size=12301) 2024-12-03T17:18:49,402 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T17:18:49,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-03T17:18:49,805 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/f4434a9c6ae3424cad3d8adb62ce708a 2024-12-03T17:18:49,836 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/2f75dc1a4fe649d8980fc60d486d6426 is 50, key is test_row_0/C:col10/1733246320614/Put/seqid=0 2024-12-03T17:18:49,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742460_1636 (size=12301) 2024-12-03T17:18:49,874 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/2f75dc1a4fe649d8980fc60d486d6426 2024-12-03T17:18:49,898 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/A/55324c50fbfc42838b819a9d45a8ec72 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/55324c50fbfc42838b819a9d45a8ec72 2024-12-03T17:18:49,902 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/55324c50fbfc42838b819a9d45a8ec72, entries=150, sequenceid=304, filesize=30.5 K 2024-12-03T17:18:49,903 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/B/f4434a9c6ae3424cad3d8adb62ce708a as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/f4434a9c6ae3424cad3d8adb62ce708a 2024-12-03T17:18:49,911 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/f4434a9c6ae3424cad3d8adb62ce708a, entries=150, sequenceid=304, filesize=12.0 K 2024-12-03T17:18:49,912 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/.tmp/C/2f75dc1a4fe649d8980fc60d486d6426 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/2f75dc1a4fe649d8980fc60d486d6426 2024-12-03T17:18:49,919 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/2f75dc1a4fe649d8980fc60d486d6426, entries=150, sequenceid=304, filesize=12.0 K 2024-12-03T17:18:49,928 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 3e5f45a8f2579c6bf86ccea0cc76d7f9 in 1440ms, sequenceid=304, compaction requested=true 2024-12-03T17:18:49,935 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/da44c70085c84a4e8f1091f42225b968, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/89222cda79534edcb639e27ba5eaaf9c, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/2b11daa95f2c4d318052b7aeafee9bf0, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/7dfe05e5b68d4778b7445b91f707380b, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/18ef0e4c158545fdb10e7a4eb4091db9, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/901459cabfdb46588ca60b40109172cf, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/ad4b5c02c88e481f9dd0a3ed9b222efd, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/a35336f06b19419ca2d77ea98304ec26, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/84df52f6296e4cd690ac87dfed412292, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/4bd4ef0534d749f3bd763c66236a73a8, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/158f9f06ad7f4c3ba01a0f99522d4dc3, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/d0fc1619dcc74efb88cb697af66bc562, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/d099eda1d9d44fc68cce3682cc9c697a, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/ec0437f3aab74089b184cc4eb4e046fe, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/2d948ec6cc1641a7b02fa50d41c77f29, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/714da794ca0c4f0c8bc6af8470ce96f0, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/10089801d0c34e3bb4e12349bd9dfdf9, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/ca14b32cbc16425481f860c80947712e] to archive 2024-12-03T17:18:49,941 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T17:18:49,953 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/da44c70085c84a4e8f1091f42225b968 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/da44c70085c84a4e8f1091f42225b968 2024-12-03T17:18:49,969 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/89222cda79534edcb639e27ba5eaaf9c to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/89222cda79534edcb639e27ba5eaaf9c 2024-12-03T17:18:49,971 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/2b11daa95f2c4d318052b7aeafee9bf0 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/2b11daa95f2c4d318052b7aeafee9bf0 2024-12-03T17:18:49,972 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/7dfe05e5b68d4778b7445b91f707380b to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/7dfe05e5b68d4778b7445b91f707380b 2024-12-03T17:18:49,973 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/18ef0e4c158545fdb10e7a4eb4091db9 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/18ef0e4c158545fdb10e7a4eb4091db9 2024-12-03T17:18:49,974 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/901459cabfdb46588ca60b40109172cf to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/901459cabfdb46588ca60b40109172cf 2024-12-03T17:18:49,991 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/ad4b5c02c88e481f9dd0a3ed9b222efd to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/ad4b5c02c88e481f9dd0a3ed9b222efd 2024-12-03T17:18:50,002 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/a35336f06b19419ca2d77ea98304ec26 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/a35336f06b19419ca2d77ea98304ec26 2024-12-03T17:18:50,014 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/84df52f6296e4cd690ac87dfed412292 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/84df52f6296e4cd690ac87dfed412292 2024-12-03T17:18:50,017 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/4bd4ef0534d749f3bd763c66236a73a8 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/4bd4ef0534d749f3bd763c66236a73a8 2024-12-03T17:18:50,030 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/158f9f06ad7f4c3ba01a0f99522d4dc3 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/158f9f06ad7f4c3ba01a0f99522d4dc3 2024-12-03T17:18:50,034 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/d0fc1619dcc74efb88cb697af66bc562 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/d0fc1619dcc74efb88cb697af66bc562 2024-12-03T17:18:50,042 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/d099eda1d9d44fc68cce3682cc9c697a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/d099eda1d9d44fc68cce3682cc9c697a 2024-12-03T17:18:50,055 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/ec0437f3aab74089b184cc4eb4e046fe to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/ec0437f3aab74089b184cc4eb4e046fe 2024-12-03T17:18:50,058 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/2d948ec6cc1641a7b02fa50d41c77f29 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/2d948ec6cc1641a7b02fa50d41c77f29 2024-12-03T17:18:50,075 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/714da794ca0c4f0c8bc6af8470ce96f0 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/714da794ca0c4f0c8bc6af8470ce96f0 2024-12-03T17:18:50,098 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/10089801d0c34e3bb4e12349bd9dfdf9 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/10089801d0c34e3bb4e12349bd9dfdf9 2024-12-03T17:18:50,110 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/ca14b32cbc16425481f860c80947712e to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/ca14b32cbc16425481f860c80947712e 2024-12-03T17:18:50,122 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/06518fcfd6af4b57aff956adb3201ea9, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/974766641dcc44cd996e916f9c33e016, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/8d8873197a5a446ea8d2b378cb432ffd, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/016a7d91fc49421b91a3fc68c25d5192, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/cabdec57535a431284b3e54e5766c2cc, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/19efac42b5904a6ea091de0770f027d1, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/86b9baa9465d403fbfbc892c018d78dc, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/69738a468bd341c2946ebbdda1064307, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/07f451c0a55c4f398131bea18b17c171, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/7aa06e2d24ca4640b966889a17eb4dd9, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/0e005ddf818f4da3813796b18828156a, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/29751bc810414d5d9c69e648609a86eb, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/8b31079b0a5c4202a43a68316da8d380, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/ca3dc74b0b014a8da08cb121d4465a31, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/73a19e7e0edc4b6f9e4d5fe8e875388f, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/4b67165ebdee4ce5af9113e8541bf7d0, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/f33ac27fc61c4feaac7754c0145f1759, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/48ae7a1f49b9436eafb7e2e4623a86f7] to archive 2024-12-03T17:18:50,127 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T17:18:50,150 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/06518fcfd6af4b57aff956adb3201ea9 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/06518fcfd6af4b57aff956adb3201ea9 2024-12-03T17:18:50,163 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/974766641dcc44cd996e916f9c33e016 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/974766641dcc44cd996e916f9c33e016 2024-12-03T17:18:50,175 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/8d8873197a5a446ea8d2b378cb432ffd to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/8d8873197a5a446ea8d2b378cb432ffd 2024-12-03T17:18:50,177 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/016a7d91fc49421b91a3fc68c25d5192 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/016a7d91fc49421b91a3fc68c25d5192 2024-12-03T17:18:50,195 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/cabdec57535a431284b3e54e5766c2cc to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/cabdec57535a431284b3e54e5766c2cc 2024-12-03T17:18:50,210 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/19efac42b5904a6ea091de0770f027d1 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/19efac42b5904a6ea091de0770f027d1 2024-12-03T17:18:50,216 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/86b9baa9465d403fbfbc892c018d78dc to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/86b9baa9465d403fbfbc892c018d78dc 2024-12-03T17:18:50,222 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/69738a468bd341c2946ebbdda1064307 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/69738a468bd341c2946ebbdda1064307 2024-12-03T17:18:50,224 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/07f451c0a55c4f398131bea18b17c171 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/07f451c0a55c4f398131bea18b17c171 2024-12-03T17:18:50,225 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/7aa06e2d24ca4640b966889a17eb4dd9 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/7aa06e2d24ca4640b966889a17eb4dd9 2024-12-03T17:18:50,226 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/0e005ddf818f4da3813796b18828156a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/0e005ddf818f4da3813796b18828156a 2024-12-03T17:18:50,227 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/29751bc810414d5d9c69e648609a86eb to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/29751bc810414d5d9c69e648609a86eb 2024-12-03T17:18:50,229 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/8b31079b0a5c4202a43a68316da8d380 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/8b31079b0a5c4202a43a68316da8d380 2024-12-03T17:18:50,230 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/ca3dc74b0b014a8da08cb121d4465a31 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/ca3dc74b0b014a8da08cb121d4465a31 2024-12-03T17:18:50,231 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/73a19e7e0edc4b6f9e4d5fe8e875388f to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/73a19e7e0edc4b6f9e4d5fe8e875388f 2024-12-03T17:18:50,232 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/4b67165ebdee4ce5af9113e8541bf7d0 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/4b67165ebdee4ce5af9113e8541bf7d0 2024-12-03T17:18:50,233 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/f33ac27fc61c4feaac7754c0145f1759 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/f33ac27fc61c4feaac7754c0145f1759 2024-12-03T17:18:50,235 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/48ae7a1f49b9436eafb7e2e4623a86f7 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/48ae7a1f49b9436eafb7e2e4623a86f7 2024-12-03T17:18:50,238 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/1181b19357214873a9abc1551db1f38e, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/5a0b591b597f452caaadd2c02195e3aa, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/477419cffdba468c8fda655cb669b4ae, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/492ec8369f5f4eb591b6969aede211b4, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/fe22d51cc264468c91d2e93055a05acf, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/09c26bcff324486c9290ce4a3ee8a9a8, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/bacea7f9d98f4e399a7af4c655dfce0d, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/3401c18fa29c42379c34ce70e2684e5f, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/a95eb3f025b14d168ad619ebe0afb0aa, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/5f68fdf67b8e4f9db4d917b44026058e, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/9c48f254cbad4ffaa1ca82571032c56b, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/803719d3635947ccb29019f984071d54, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/d0786d5332d841bc942184191a12e3e6, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/7f901f1c117e43a9bcd7dd23c1b71a93, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/3e04cae0db914220becbae247176ae67, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/f7aa2098da7649d69890dc5b1b395ec8, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/597e352b7c1e44d5b887a9e0e8fd4f9e, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/0f960c382d0c47e7bd4024bb71eb7b70] to archive 2024-12-03T17:18:50,240 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T17:18:50,242 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/1181b19357214873a9abc1551db1f38e to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/1181b19357214873a9abc1551db1f38e 2024-12-03T17:18:50,243 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/5a0b591b597f452caaadd2c02195e3aa to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/5a0b591b597f452caaadd2c02195e3aa 2024-12-03T17:18:50,244 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/477419cffdba468c8fda655cb669b4ae to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/477419cffdba468c8fda655cb669b4ae 2024-12-03T17:18:50,245 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/492ec8369f5f4eb591b6969aede211b4 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/492ec8369f5f4eb591b6969aede211b4 2024-12-03T17:18:50,246 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/fe22d51cc264468c91d2e93055a05acf to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/fe22d51cc264468c91d2e93055a05acf 2024-12-03T17:18:50,248 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/09c26bcff324486c9290ce4a3ee8a9a8 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/09c26bcff324486c9290ce4a3ee8a9a8 2024-12-03T17:18:50,249 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/bacea7f9d98f4e399a7af4c655dfce0d to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/bacea7f9d98f4e399a7af4c655dfce0d 2024-12-03T17:18:50,250 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/3401c18fa29c42379c34ce70e2684e5f to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/3401c18fa29c42379c34ce70e2684e5f 2024-12-03T17:18:50,251 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/a95eb3f025b14d168ad619ebe0afb0aa to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/a95eb3f025b14d168ad619ebe0afb0aa 2024-12-03T17:18:50,252 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/5f68fdf67b8e4f9db4d917b44026058e to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/5f68fdf67b8e4f9db4d917b44026058e 2024-12-03T17:18:50,253 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/9c48f254cbad4ffaa1ca82571032c56b to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/9c48f254cbad4ffaa1ca82571032c56b 2024-12-03T17:18:50,254 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/803719d3635947ccb29019f984071d54 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/803719d3635947ccb29019f984071d54 2024-12-03T17:18:50,255 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/d0786d5332d841bc942184191a12e3e6 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/d0786d5332d841bc942184191a12e3e6 2024-12-03T17:18:50,256 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/7f901f1c117e43a9bcd7dd23c1b71a93 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/7f901f1c117e43a9bcd7dd23c1b71a93 2024-12-03T17:18:50,258 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/3e04cae0db914220becbae247176ae67 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/3e04cae0db914220becbae247176ae67 2024-12-03T17:18:50,264 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/f7aa2098da7649d69890dc5b1b395ec8 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/f7aa2098da7649d69890dc5b1b395ec8 2024-12-03T17:18:50,265 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/597e352b7c1e44d5b887a9e0e8fd4f9e to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/597e352b7c1e44d5b887a9e0e8fd4f9e 2024-12-03T17:18:50,266 DEBUG [StoreCloser-TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/0f960c382d0c47e7bd4024bb71eb7b70 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/0f960c382d0c47e7bd4024bb71eb7b70 2024-12-03T17:18:50,303 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/recovered.edits/307.seqid, newMaxSeqId=307, maxSeqId=4 2024-12-03T17:18:50,304 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. 2024-12-03T17:18:50,304 DEBUG [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1676): Region close journal for 3e5f45a8f2579c6bf86ccea0cc76d7f9: Waiting for close lock at 1733246328488Running coprocessor pre-close hooks at 1733246328488Disabling compacts and flushes for region at 1733246328488Disabling writes for close at 1733246328488Obtaining lock to block concurrent updates at 1733246328488Preparing flush snapshotting stores in 3e5f45a8f2579c6bf86ccea0cc76d7f9 at 1733246328488Finished memstore snapshotting TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733246328489 (+1 ms)Flushing stores of TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9. at 1733246328489Flushing 3e5f45a8f2579c6bf86ccea0cc76d7f9/A: creating writer at 1733246328489Flushing mob file 3e5f45a8f2579c6bf86ccea0cc76d7f9/A: appending metadata at 1733246328514 (+25 ms)Flushing mob file 3e5f45a8f2579c6bf86ccea0cc76d7f9/A: closing flushed file at 1733246328514Flushing 3e5f45a8f2579c6bf86ccea0cc76d7f9/A: appending metadata at 1733246328958 (+444 ms)Flushing 3e5f45a8f2579c6bf86ccea0cc76d7f9/A: closing flushed file at 1733246328959 (+1 ms)Flushing 3e5f45a8f2579c6bf86ccea0cc76d7f9/B: creating writer at 1733246329375 (+416 ms)Flushing 3e5f45a8f2579c6bf86ccea0cc76d7f9/B: appending metadata at 1733246329378 (+3 ms)Flushing 3e5f45a8f2579c6bf86ccea0cc76d7f9/B: closing flushed file at 1733246329378Flushing 3e5f45a8f2579c6bf86ccea0cc76d7f9/C: creating writer at 1733246329821 (+443 ms)Flushing 3e5f45a8f2579c6bf86ccea0cc76d7f9/C: appending metadata at 1733246329835 (+14 ms)Flushing 3e5f45a8f2579c6bf86ccea0cc76d7f9/C: closing flushed file at 1733246329836 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@152f633: reopening flushed file at 1733246329894 (+58 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2fde9080: reopening flushed file at 1733246329903 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d44475b: reopening flushed file at 1733246329911 (+8 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 3e5f45a8f2579c6bf86ccea0cc76d7f9 in 1440ms, sequenceid=304, compaction requested=true at 1733246329928 (+17 ms)Writing region close event to WAL at 1733246330286 (+358 ms)Running coprocessor post-close hooks at 1733246330304 (+18 ms)Closed at 1733246330304 2024-12-03T17:18:50,319 INFO [RS_CLOSE_REGION-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] handler.UnassignRegionHandler(157): Closed 3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:50,329 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=158 updating hbase:meta row=3e5f45a8f2579c6bf86ccea0cc76d7f9, regionState=CLOSED 2024-12-03T17:18:50,339 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=159, ppid=158, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 because future has completed 2024-12-03T17:18:50,350 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-12-03T17:18:50,351 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; CloseRegionProcedure 3e5f45a8f2579c6bf86ccea0cc76d7f9, server=27e0f852a164,41167,1733246151593 in 2.0060 sec 2024-12-03T17:18:50,371 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=158, resume processing ppid=157 2024-12-03T17:18:50,371 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, ppid=157, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3e5f45a8f2579c6bf86ccea0cc76d7f9, UNASSIGN in 2.0230 sec 2024-12-03T17:18:50,377 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733246330376"}]},"ts":"1733246330376"} 2024-12-03T17:18:50,377 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-12-03T17:18:50,377 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.0470 sec 2024-12-03T17:18:50,406 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-03T17:18:50,406 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-03T17:18:50,418 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.1060 sec 2024-12-03T17:18:50,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-03T17:18:50,491 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:18:50,492 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-03T17:18:50,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T17:18:50,499 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=160, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T17:18:50,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-03T17:18:50,502 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=160, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T17:18:50,512 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:50,535 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A, FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B, FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C, FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/recovered.edits] 2024-12-03T17:18:50,541 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/4b805274fdba4d4ea0b2e385d7f9b9a9 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/4b805274fdba4d4ea0b2e385d7f9b9a9 2024-12-03T17:18:50,542 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/55324c50fbfc42838b819a9d45a8ec72 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/55324c50fbfc42838b819a9d45a8ec72 2024-12-03T17:18:50,554 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/9d43a76483a24ef18f188e70c03605ea to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/9d43a76483a24ef18f188e70c03605ea 2024-12-03T17:18:50,563 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/e557935fcbfa46ff95bc67aabd473280 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/A/e557935fcbfa46ff95bc67aabd473280 2024-12-03T17:18:50,586 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/1a87d4a459354a32a6ed4ff83a6e1c3d to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/1a87d4a459354a32a6ed4ff83a6e1c3d 2024-12-03T17:18:50,593 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/39f8d67dd48e40ba8ca6bad4120b5441 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/39f8d67dd48e40ba8ca6bad4120b5441 2024-12-03T17:18:50,605 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/be63c862379f41969bbbc8106f244141 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/be63c862379f41969bbbc8106f244141 2024-12-03T17:18:50,610 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/f4434a9c6ae3424cad3d8adb62ce708a to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/B/f4434a9c6ae3424cad3d8adb62ce708a 2024-12-03T17:18:50,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-03T17:18:50,635 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/2f75dc1a4fe649d8980fc60d486d6426 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/2f75dc1a4fe649d8980fc60d486d6426 2024-12-03T17:18:50,638 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/46b8b4f3971c4cb8a2d1e2ed8dfe737c to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/46b8b4f3971c4cb8a2d1e2ed8dfe737c 2024-12-03T17:18:50,640 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/52405f55fdeb4c5190423fd27ee8a366 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/52405f55fdeb4c5190423fd27ee8a366 2024-12-03T17:18:50,641 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/8dcb6ed9a48c4b5ab65ea4beb816ad99 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/C/8dcb6ed9a48c4b5ab65ea4beb816ad99 2024-12-03T17:18:50,663 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/recovered.edits/307.seqid to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9/recovered.edits/307.seqid 2024-12-03T17:18:50,666 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/default/TestAcidGuarantees/3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:50,667 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-03T17:18:50,668 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-03T17:18:50,668 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-03T17:18:50,673 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412030270750143124e8d8d5ae07dff6bc824_3e5f45a8f2579c6bf86ccea0cc76d7f9 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412030270750143124e8d8d5ae07dff6bc824_3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:50,679 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203044153809afe433f96f777b8bf68c885_3e5f45a8f2579c6bf86ccea0cc76d7f9 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203044153809afe433f96f777b8bf68c885_3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:50,697 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412030acec4eafc4945a9b4d9a7f96bfd678e_3e5f45a8f2579c6bf86ccea0cc76d7f9 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412030acec4eafc4945a9b4d9a7f96bfd678e_3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:50,717 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412031699372b192b43569d36f71ddde39c5f_3e5f45a8f2579c6bf86ccea0cc76d7f9 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412031699372b192b43569d36f71ddde39c5f_3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:50,727 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203286097c1986c46f2b57e6727924c7a5a_3e5f45a8f2579c6bf86ccea0cc76d7f9 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203286097c1986c46f2b57e6727924c7a5a_3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:50,734 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412032a5b14bdfdc74c3bbbda00f5cc212ef4_3e5f45a8f2579c6bf86ccea0cc76d7f9 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412032a5b14bdfdc74c3bbbda00f5cc212ef4_3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:50,738 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203304d189c5e544b6cba509893a2be9e8a_3e5f45a8f2579c6bf86ccea0cc76d7f9 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203304d189c5e544b6cba509893a2be9e8a_3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:50,747 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203574cabd537314f709ba6a2aa816f4590_3e5f45a8f2579c6bf86ccea0cc76d7f9 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203574cabd537314f709ba6a2aa816f4590_3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:50,754 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036f512a0b1e854af4a4480e736d59b7a4_3e5f45a8f2579c6bf86ccea0cc76d7f9 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412036f512a0b1e854af4a4480e736d59b7a4_3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:50,764 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203719170cf92df4499a9dc8ce1150a3df8_3e5f45a8f2579c6bf86ccea0cc76d7f9 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203719170cf92df4499a9dc8ce1150a3df8_3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:50,772 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203952c49a72c1a434f898f83e90da760fd_3e5f45a8f2579c6bf86ccea0cc76d7f9 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203952c49a72c1a434f898f83e90da760fd_3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:50,790 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120395692dfcaef54da2b8fb78ac603eb81c_3e5f45a8f2579c6bf86ccea0cc76d7f9 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120395692dfcaef54da2b8fb78ac603eb81c_3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:50,805 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120396d5f2a0f408474a8969b08dc3c55201_3e5f45a8f2579c6bf86ccea0cc76d7f9 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120396d5f2a0f408474a8969b08dc3c55201_3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:50,814 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039d116831a10e42ddb7d9597151a4178f_3e5f45a8f2579c6bf86ccea0cc76d7f9 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412039d116831a10e42ddb7d9597151a4178f_3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:50,823 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d6537a7e7f144e1d887ee934b396d6ad_3e5f45a8f2579c6bf86ccea0cc76d7f9 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203d6537a7e7f144e1d887ee934b396d6ad_3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:50,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-03T17:18:50,830 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ecf4fa6dfe0442b28fd387e1a4ba389e_3e5f45a8f2579c6bf86ccea0cc76d7f9 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203ecf4fa6dfe0442b28fd387e1a4ba389e_3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:50,835 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203fdb5c1255e2141a6a703cb10cd09a6d6_3e5f45a8f2579c6bf86ccea0cc76d7f9 to hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241203fdb5c1255e2141a6a703cb10cd09a6d6_3e5f45a8f2579c6bf86ccea0cc76d7f9 2024-12-03T17:18:50,845 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-03T17:18:50,870 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=160, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T17:18:50,873 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-03T17:18:50,878 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-03T17:18:50,888 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=160, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T17:18:50,888 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-03T17:18:50,889 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733246330888"}]},"ts":"9223372036854775807"} 2024-12-03T17:18:50,892 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-03T17:18:50,892 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 3e5f45a8f2579c6bf86ccea0cc76d7f9, NAME => 'TestAcidGuarantees,,1733246298964.3e5f45a8f2579c6bf86ccea0cc76d7f9.', STARTKEY => '', ENDKEY => ''}] 2024-12-03T17:18:50,892 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-03T17:18:50,892 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733246330892"}]},"ts":"9223372036854775807"} 2024-12-03T17:18:50,894 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-03T17:18:50,898 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=160, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-03T17:18:50,904 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 406 msec 2024-12-03T17:18:51,122 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-03T17:18:51,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38523 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-03T17:18:51,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-03T17:18:51,137 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-03T17:18:51,154 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247 (was 250), OpenFileDescriptor=443 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1268 (was 1505), ProcessCount=11 (was 11), AvailableMemoryMB=4384 (was 4175) - AvailableMemoryMB LEAK? - 2024-12-03T17:18:51,154 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-03T17:18:51,154 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T17:18:51,154 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T17:18:51,154 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:51,155 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:51,155 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T17:18:51,155 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-03T17:18:51,155 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1258311727, stopped=false 2024-12-03T17:18:51,155 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=27e0f852a164,38523,1733246150595 2024-12-03T17:18:51,218 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41167-0x1010d48f3a20001, quorum=127.0.0.1:56976, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-03T17:18:51,218 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38523-0x1010d48f3a20000, quorum=127.0.0.1:56976, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-03T17:18:51,218 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41167-0x1010d48f3a20001, quorum=127.0.0.1:56976, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T17:18:51,218 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38523-0x1010d48f3a20000, quorum=127.0.0.1:56976, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T17:18:51,219 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T17:18:51,220 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38523-0x1010d48f3a20000, quorum=127.0.0.1:56976, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T17:18:51,220 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T17:18:51,221 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T17:18:51,221 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:51,223 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '27e0f852a164,41167,1733246151593' ***** 2024-12-03T17:18:51,223 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-03T17:18:51,223 INFO [RS:0;27e0f852a164:41167 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-03T17:18:51,223 INFO [RS:0;27e0f852a164:41167 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-03T17:18:51,224 INFO [RS:0;27e0f852a164:41167 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-03T17:18:51,224 INFO [RS:0;27e0f852a164:41167 {}] regionserver.HRegionServer(959): stopping server 27e0f852a164,41167,1733246151593 2024-12-03T17:18:51,224 INFO [RS:0;27e0f852a164:41167 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T17:18:51,224 INFO [RS:0;27e0f852a164:41167 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;27e0f852a164:41167. 2024-12-03T17:18:51,224 DEBUG [RS:0;27e0f852a164:41167 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T17:18:51,224 DEBUG [RS:0;27e0f852a164:41167 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:51,224 INFO [RS:0;27e0f852a164:41167 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-03T17:18:51,224 INFO [RS:0;27e0f852a164:41167 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-03T17:18:51,224 INFO [RS:0;27e0f852a164:41167 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-03T17:18:51,224 INFO [RS:0;27e0f852a164:41167 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-03T17:18:51,225 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-03T17:18:51,226 INFO [RS:0;27e0f852a164:41167 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-03T17:18:51,226 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41167-0x1010d48f3a20001, quorum=127.0.0.1:56976, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T17:18:51,227 DEBUG [RS:0;27e0f852a164:41167 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-03T17:18:51,227 DEBUG [RS:0;27e0f852a164:41167 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-03T17:18:51,227 DEBUG [RS_CLOSE_META-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-03T17:18:51,227 INFO [RS_CLOSE_META-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-03T17:18:51,227 DEBUG [RS_CLOSE_META-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-03T17:18:51,227 DEBUG [RS_CLOSE_META-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-03T17:18:51,227 DEBUG [RS_CLOSE_META-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-03T17:18:51,228 INFO [RS_CLOSE_META-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-12-03T17:18:51,259 INFO [regionserver/27e0f852a164:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T17:18:51,271 DEBUG [RS_CLOSE_META-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/hbase/meta/1588230740/.tmp/info/c6135dd846b94335950929ef134bae1f is 95, key is TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55./info:/1733246182804/DeleteFamily/seqid=0 2024-12-03T17:18:51,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742461_1637 (size=6514) 2024-12-03T17:18:51,427 DEBUG [RS:0;27e0f852a164:41167 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-03T17:18:51,627 DEBUG [RS:0;27e0f852a164:41167 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-03T17:18:51,678 INFO [RS_CLOSE_META-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/hbase/meta/1588230740/.tmp/info/c6135dd846b94335950929ef134bae1f 2024-12-03T17:18:51,730 DEBUG [RS_CLOSE_META-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/hbase/meta/1588230740/.tmp/ns/9dc5e02b207442f79d5931de8eb4b14f is 93, key is TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55./ns:/1733246182736/DeleteFamily/seqid=0 2024-12-03T17:18:51,746 INFO [regionserver/27e0f852a164:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-03T17:18:51,746 INFO [regionserver/27e0f852a164:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-03T17:18:51,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742462_1638 (size=5906) 2024-12-03T17:18:51,828 DEBUG [RS:0;27e0f852a164:41167 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-03T17:18:52,028 DEBUG [RS:0;27e0f852a164:41167 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-03T17:18:52,190 INFO [RS_CLOSE_META-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/hbase/meta/1588230740/.tmp/ns/9dc5e02b207442f79d5931de8eb4b14f 2024-12-03T17:18:52,229 DEBUG [RS:0;27e0f852a164:41167 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-03T17:18:52,232 DEBUG [RS_CLOSE_META-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/hbase/meta/1588230740/.tmp/rep_barrier/d4751ab9642c41a1b5593faf7ec1182e is 102, key is TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55./rep_barrier:/1733246182736/DeleteFamily/seqid=0 2024-12-03T17:18:52,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742463_1639 (size=6025) 2024-12-03T17:18:52,271 INFO [RS_CLOSE_META-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/hbase/meta/1588230740/.tmp/rep_barrier/d4751ab9642c41a1b5593faf7ec1182e 2024-12-03T17:18:52,359 DEBUG [RS_CLOSE_META-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/hbase/meta/1588230740/.tmp/table/d74d4c73e61941b087eefa0a6fb1a9aa is 96, key is TestAcidGuarantees,,1733246156562.e1e0418f3d66f886e020279128ac6f55./table:/1733246182736/DeleteFamily/seqid=0 2024-12-03T17:18:52,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742464_1640 (size=5936) 2024-12-03T17:18:52,429 DEBUG [RS:0;27e0f852a164:41167 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-03T17:18:52,630 DEBUG [RS:0;27e0f852a164:41167 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-03T17:18:52,827 INFO [RS_CLOSE_META-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/hbase/meta/1588230740/.tmp/table/d74d4c73e61941b087eefa0a6fb1a9aa 2024-12-03T17:18:52,833 DEBUG [RS:0;27e0f852a164:41167 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-03T17:18:52,852 DEBUG [RS_CLOSE_META-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/hbase/meta/1588230740/.tmp/info/c6135dd846b94335950929ef134bae1f as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/hbase/meta/1588230740/info/c6135dd846b94335950929ef134bae1f 2024-12-03T17:18:52,857 INFO [RS_CLOSE_META-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/hbase/meta/1588230740/info/c6135dd846b94335950929ef134bae1f, entries=12, sequenceid=90, filesize=6.4 K 2024-12-03T17:18:52,858 DEBUG [RS_CLOSE_META-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/hbase/meta/1588230740/.tmp/ns/9dc5e02b207442f79d5931de8eb4b14f as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/hbase/meta/1588230740/ns/9dc5e02b207442f79d5931de8eb4b14f 2024-12-03T17:18:52,862 INFO [RS_CLOSE_META-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/hbase/meta/1588230740/ns/9dc5e02b207442f79d5931de8eb4b14f, entries=8, sequenceid=90, filesize=5.8 K 2024-12-03T17:18:52,863 DEBUG [RS_CLOSE_META-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/hbase/meta/1588230740/.tmp/rep_barrier/d4751ab9642c41a1b5593faf7ec1182e as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/hbase/meta/1588230740/rep_barrier/d4751ab9642c41a1b5593faf7ec1182e 2024-12-03T17:18:52,867 INFO [RS_CLOSE_META-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/hbase/meta/1588230740/rep_barrier/d4751ab9642c41a1b5593faf7ec1182e, entries=6, sequenceid=90, filesize=5.9 K 2024-12-03T17:18:52,867 DEBUG [RS_CLOSE_META-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/hbase/meta/1588230740/.tmp/table/d74d4c73e61941b087eefa0a6fb1a9aa as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/hbase/meta/1588230740/table/d74d4c73e61941b087eefa0a6fb1a9aa 2024-12-03T17:18:52,871 INFO [RS_CLOSE_META-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/hbase/meta/1588230740/table/d74d4c73e61941b087eefa0a6fb1a9aa, entries=7, sequenceid=90, filesize=5.8 K 2024-12-03T17:18:52,883 INFO [RS_CLOSE_META-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1656ms, sequenceid=90, compaction requested=false 2024-12-03T17:18:52,966 DEBUG [RS_CLOSE_META-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-12-03T17:18:52,972 DEBUG [RS_CLOSE_META-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-03T17:18:52,972 INFO [RS_CLOSE_META-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-03T17:18:52,973 DEBUG [RS_CLOSE_META-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733246331227Running coprocessor pre-close hooks at 1733246331227Disabling compacts and flushes for region at 1733246331227Disabling writes for close at 1733246331227Obtaining lock to block concurrent updates at 1733246331228 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1733246331228Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1733246331228Flushing stores of hbase:meta,,1.1588230740 at 1733246331230 (+2 ms)Flushing 1588230740/info: creating writer at 1733246331231 (+1 ms)Flushing 1588230740/info: appending metadata at 1733246331270 (+39 ms)Flushing 1588230740/info: closing flushed file at 1733246331270Flushing 1588230740/ns: creating writer at 1733246331690 (+420 ms)Flushing 1588230740/ns: appending metadata at 1733246331722 (+32 ms)Flushing 1588230740/ns: closing flushed file at 1733246331722Flushing 1588230740/rep_barrier: creating writer at 1733246332200 (+478 ms)Flushing 1588230740/rep_barrier: appending metadata at 1733246332232 (+32 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1733246332232Flushing 1588230740/table: creating writer at 1733246332300 (+68 ms)Flushing 1588230740/table: appending metadata at 1733246332358 (+58 ms)Flushing 1588230740/table: closing flushed file at 1733246332358Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@77bf721f: reopening flushed file at 1733246332850 (+492 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@94210cf: reopening flushed file at 1733246332857 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4907f68a: reopening flushed file at 1733246332862 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16576801: reopening flushed file at 1733246332867 (+5 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1656ms, sequenceid=90, compaction requested=false at 1733246332883 (+16 ms)Writing region close event to WAL at 1733246332942 (+59 ms)Running coprocessor post-close hooks at 1733246332972 (+30 ms)Closed at 1733246332972 2024-12-03T17:18:52,973 DEBUG [RS_CLOSE_META-regionserver/27e0f852a164:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-03T17:18:53,034 INFO [RS:0;27e0f852a164:41167 {}] regionserver.HRegionServer(976): stopping server 27e0f852a164,41167,1733246151593; all regions closed. 2024-12-03T17:18:53,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741834_1010 (size=25344) 2024-12-03T17:18:53,071 DEBUG [RS:0;27e0f852a164:41167 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/oldWALs 2024-12-03T17:18:53,071 INFO [RS:0;27e0f852a164:41167 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 27e0f852a164%2C41167%2C1733246151593.meta:.meta(num 1733246155844) 2024-12-03T17:18:53,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741833_1009 (size=18062642) 2024-12-03T17:18:53,092 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/WALs/27e0f852a164,41167,1733246151593/27e0f852a164%2C41167%2C1733246151593.1733246154949 not finished, retry = 0 2024-12-03T17:18:53,210 DEBUG [RS:0;27e0f852a164:41167 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/oldWALs 2024-12-03T17:18:53,210 INFO [RS:0;27e0f852a164:41167 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 27e0f852a164%2C41167%2C1733246151593:(num 1733246154949) 2024-12-03T17:18:53,210 DEBUG [RS:0;27e0f852a164:41167 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T17:18:53,211 INFO [RS:0;27e0f852a164:41167 {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T17:18:53,211 INFO [RS:0;27e0f852a164:41167 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T17:18:53,211 INFO [RS:0;27e0f852a164:41167 {}] hbase.ChoreService(370): Chore service for: regionserver/27e0f852a164:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-03T17:18:53,211 INFO [RS:0;27e0f852a164:41167 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T17:18:53,211 INFO [regionserver/27e0f852a164:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T17:18:53,212 INFO [RS:0;27e0f852a164:41167 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:41167 2024-12-03T17:18:53,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41167-0x1010d48f3a20001, quorum=127.0.0.1:56976, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/27e0f852a164,41167,1733246151593 2024-12-03T17:18:53,229 INFO [RS:0;27e0f852a164:41167 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T17:18:53,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38523-0x1010d48f3a20000, quorum=127.0.0.1:56976, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T17:18:53,237 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [27e0f852a164,41167,1733246151593] 2024-12-03T17:18:53,250 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/27e0f852a164,41167,1733246151593 already deleted, retry=false 2024-12-03T17:18:53,251 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 27e0f852a164,41167,1733246151593 expired; onlineServers=0 2024-12-03T17:18:53,251 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '27e0f852a164,38523,1733246150595' ***** 2024-12-03T17:18:53,251 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-03T17:18:53,252 INFO [M:0;27e0f852a164:38523 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T17:18:53,252 INFO [M:0;27e0f852a164:38523 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T17:18:53,252 DEBUG [M:0;27e0f852a164:38523 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-03T17:18:53,252 DEBUG [M:0;27e0f852a164:38523 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-03T17:18:53,252 DEBUG [master/27e0f852a164:0:becomeActiveMaster-HFileCleaner.large.0-1733246154393 {}] cleaner.HFileCleaner(306): Exit Thread[master/27e0f852a164:0:becomeActiveMaster-HFileCleaner.large.0-1733246154393,5,FailOnTimeoutGroup] 2024-12-03T17:18:53,252 INFO [M:0;27e0f852a164:38523 {}] hbase.ChoreService(370): Chore service for: master/27e0f852a164:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-03T17:18:53,252 INFO [M:0;27e0f852a164:38523 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T17:18:53,252 DEBUG [M:0;27e0f852a164:38523 {}] master.HMaster(1795): Stopping service threads 2024-12-03T17:18:53,252 INFO [M:0;27e0f852a164:38523 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-03T17:18:53,252 INFO [M:0;27e0f852a164:38523 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T17:18:53,253 ERROR [M:0;27e0f852a164:38523 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-12-03T17:18:53,253 INFO [M:0;27e0f852a164:38523 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T17:18:53,254 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-03T17:18:53,254 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-03T17:18:53,254 DEBUG [master/27e0f852a164:0:becomeActiveMaster-HFileCleaner.small.0-1733246154395 {}] cleaner.HFileCleaner(306): Exit Thread[master/27e0f852a164:0:becomeActiveMaster-HFileCleaner.small.0-1733246154395,5,FailOnTimeoutGroup] 2024-12-03T17:18:53,265 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38523-0x1010d48f3a20000, quorum=127.0.0.1:56976, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-03T17:18:53,265 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38523-0x1010d48f3a20000, quorum=127.0.0.1:56976, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T17:18:53,267 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/master already deleted, retry=false 2024-12-03T17:18:53,267 INFO [M:0;27e0f852a164:38523 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/.lastflushedseqids 2024-12-03T17:18:53,267 DEBUG [RegionServerTracker-0 {}] master.ActiveMasterManager(353): master:38523-0x1010d48f3a20000, quorum=127.0.0.1:56976, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = NoNode for /hbase/master 2024-12-03T17:18:53,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742465_1641 (size=72) 2024-12-03T17:18:53,337 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41167-0x1010d48f3a20001, quorum=127.0.0.1:56976, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T17:18:53,337 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41167-0x1010d48f3a20001, quorum=127.0.0.1:56976, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T17:18:53,338 INFO [RS:0;27e0f852a164:41167 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T17:18:53,338 INFO [RS:0;27e0f852a164:41167 {}] regionserver.HRegionServer(1031): Exiting; stopping=27e0f852a164,41167,1733246151593; zookeeper connection closed. 2024-12-03T17:18:53,342 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@502d695 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@502d695 2024-12-03T17:18:53,343 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-03T17:18:53,693 INFO [M:0;27e0f852a164:38523 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T17:18:53,694 INFO [M:0;27e0f852a164:38523 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-03T17:18:53,694 DEBUG [M:0;27e0f852a164:38523 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-03T17:18:53,694 INFO [M:0;27e0f852a164:38523 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T17:18:53,694 DEBUG [M:0;27e0f852a164:38523 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T17:18:53,694 DEBUG [M:0;27e0f852a164:38523 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-03T17:18:53,694 DEBUG [M:0;27e0f852a164:38523 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T17:18:53,694 INFO [M:0;27e0f852a164:38523 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=757.64 KB heapSize=926.73 KB 2024-12-03T17:18:53,712 DEBUG [M:0;27e0f852a164:38523 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a433b6f96fab4ba386ad590780dcacde is 82, key is hbase:meta,,1/info:regioninfo/1733246156002/Put/seqid=0 2024-12-03T17:18:53,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742466_1642 (size=5672) 2024-12-03T17:18:53,716 INFO [M:0;27e0f852a164:38523 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2090 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a433b6f96fab4ba386ad590780dcacde 2024-12-03T17:18:53,739 DEBUG [M:0;27e0f852a164:38523 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0b92653eed7c4ef082b8f8c8b63b8876 is 2481, key is \x00\x00\x00\x00\x00\x00\x00\x89/proc:d/1733246300291/Put/seqid=0 2024-12-03T17:18:53,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742467_1643 (size=43677) 2024-12-03T17:18:53,752 INFO [M:0;27e0f852a164:38523 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=757.08 KB at sequenceid=2090 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0b92653eed7c4ef082b8f8c8b63b8876 2024-12-03T17:18:53,757 INFO [M:0;27e0f852a164:38523 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0b92653eed7c4ef082b8f8c8b63b8876 2024-12-03T17:18:53,782 DEBUG [M:0;27e0f852a164:38523 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/be5b255f19874e8da7d0ca3c2086c5c9 is 69, key is 27e0f852a164,41167,1733246151593/rs:state/1733246154542/Put/seqid=0 2024-12-03T17:18:53,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073742468_1644 (size=5156) 2024-12-03T17:18:54,187 INFO [M:0;27e0f852a164:38523 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2090 (bloomFilter=true), to=hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/be5b255f19874e8da7d0ca3c2086c5c9 2024-12-03T17:18:54,192 DEBUG [M:0;27e0f852a164:38523 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a433b6f96fab4ba386ad590780dcacde as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a433b6f96fab4ba386ad590780dcacde 2024-12-03T17:18:54,197 INFO [M:0;27e0f852a164:38523 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a433b6f96fab4ba386ad590780dcacde, entries=8, sequenceid=2090, filesize=5.5 K 2024-12-03T17:18:54,198 DEBUG [M:0;27e0f852a164:38523 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0b92653eed7c4ef082b8f8c8b63b8876 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0b92653eed7c4ef082b8f8c8b63b8876 2024-12-03T17:18:54,201 INFO [M:0;27e0f852a164:38523 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0b92653eed7c4ef082b8f8c8b63b8876 2024-12-03T17:18:54,201 INFO [M:0;27e0f852a164:38523 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0b92653eed7c4ef082b8f8c8b63b8876, entries=160, sequenceid=2090, filesize=42.7 K 2024-12-03T17:18:54,202 DEBUG [M:0;27e0f852a164:38523 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/be5b255f19874e8da7d0ca3c2086c5c9 as hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/be5b255f19874e8da7d0ca3c2086c5c9 2024-12-03T17:18:54,206 INFO [M:0;27e0f852a164:38523 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39175/user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/be5b255f19874e8da7d0ca3c2086c5c9, entries=1, sequenceid=2090, filesize=5.0 K 2024-12-03T17:18:54,208 INFO [M:0;27e0f852a164:38523 {}] regionserver.HRegion(3140): Finished flush of dataSize ~757.64 KB/775822, heapSize ~926.44 KB/948672, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 514ms, sequenceid=2090, compaction requested=false 2024-12-03T17:18:54,223 INFO [M:0;27e0f852a164:38523 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T17:18:54,223 DEBUG [M:0;27e0f852a164:38523 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733246333694Disabling compacts and flushes for region at 1733246333694Disabling writes for close at 1733246333694Obtaining lock to block concurrent updates at 1733246333694Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733246333694Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=775822, getHeapSize=948912, getOffHeapSize=0, getCellsCount=2275 at 1733246333694Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733246333695 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733246333695Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733246333712 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733246333712Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733246333722 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733246333739 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733246333739Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733246333758 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733246333782 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733246333782Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@234f44b2: reopening flushed file at 1733246334191 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@123db380: reopening flushed file at 1733246334197 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4de2188b: reopening flushed file at 1733246334202 (+5 ms)Finished flush of dataSize ~757.64 KB/775822, heapSize ~926.44 KB/948672, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 514ms, sequenceid=2090, compaction requested=false at 1733246334208 (+6 ms)Writing region close event to WAL at 1733246334222 (+14 ms)Closed at 1733246334222 2024-12-03T17:18:54,237 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/f82419d0-a496-fa1d-a548-d6ed652019b4/MasterData/WALs/27e0f852a164,38523,1733246150595/27e0f852a164%2C38523%2C1733246150595.1733246153522 not finished, retry = 0 2024-12-03T17:18:54,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38185 is added to blk_1073741830_1006 (size=912831) 2024-12-03T17:18:54,338 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T17:18:54,339 INFO [M:0;27e0f852a164:38523 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-03T17:18:54,339 INFO [M:0;27e0f852a164:38523 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:38523 2024-12-03T17:18:54,339 INFO [M:0;27e0f852a164:38523 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T17:18:54,441 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38523-0x1010d48f3a20000, quorum=127.0.0.1:56976, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T17:18:54,441 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38523-0x1010d48f3a20000, quorum=127.0.0.1:56976, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T17:18:54,441 INFO [M:0;27e0f852a164:38523 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T17:18:54,449 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T17:18:54,451 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T17:18:54,451 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T17:18:54,452 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T17:18:54,452 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f784850d-ec8c-cbd4-e545-debb5ca850cc/hadoop.log.dir/,STOPPED} 2024-12-03T17:18:54,459 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-03T17:18:54,459 WARN [BP-2063810512-172.17.0.3-1733246146518 heartbeating to localhost/127.0.0.1:39175 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-03T17:18:54,459 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-03T17:18:54,459 WARN [BP-2063810512-172.17.0.3-1733246146518 heartbeating to localhost/127.0.0.1:39175 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2063810512-172.17.0.3-1733246146518 (Datanode Uuid e44d842c-8e46-41fc-93b6-c38882841a8c) service to localhost/127.0.0.1:39175 2024-12-03T17:18:54,461 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f784850d-ec8c-cbd4-e545-debb5ca850cc/cluster_481ba349-0fa7-2aa3-a4cc-d5a666163a05/data/data1/current/BP-2063810512-172.17.0.3-1733246146518 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T17:18:54,462 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f784850d-ec8c-cbd4-e545-debb5ca850cc/cluster_481ba349-0fa7-2aa3-a4cc-d5a666163a05/data/data2/current/BP-2063810512-172.17.0.3-1733246146518 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T17:18:54,462 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-03T17:18:54,471 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-03T17:18:54,472 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T17:18:54,472 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T17:18:54,472 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T17:18:54,472 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f784850d-ec8c-cbd4-e545-debb5ca850cc/hadoop.log.dir/,STOPPED} 2024-12-03T17:18:54,496 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-03T17:18:54,640 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down